summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMax Magorsch <arzano@gentoo.org>2020-03-03 21:45:00 +0100
committerMax Magorsch <arzano@gentoo.org>2020-03-03 21:45:00 +0100
commit4f1b89be9b805fbd549c86c5b49c814f836a65b0 (patch)
tree1c12725ebbf79b267d09e7f9e7700aa6dd611ed5
parentUse bundler 1.x on travis for now (diff)
downloadglsamaker-2.1.17.tar.gz
glsamaker-2.1.17.tar.bz2
glsamaker-2.1.17.zip
Fix the cve import2.1.17
The cve feeds are using json instead of xml now. The import task has been migrated to read the json feeds now. Signed-off-by: Max Magorsch <arzano@gentoo.org>
-rw-r--r--lib/tasks/cve.rake422
1 files changed, 251 insertions, 171 deletions
diff --git a/lib/tasks/cve.rake b/lib/tasks/cve.rake
index dbe06cf..4fccf47 100644
--- a/lib/tasks/cve.rake
+++ b/lib/tasks/cve.rake
@@ -1,5 +1,6 @@
# ===GLSAMaker v2
# Copyright (C) 2010–15 Alex Legler <a3li@gentoo.org>
+# Copyright (C) 2020 Max Magorsch <arzano@gentoo.org>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
@@ -7,8 +8,7 @@
# (at your option) any later version.
#
# For more information, see the LICENSE file.
-
-libs = %w( nokogiri zlib stringio )
+libs = %w( json nokogiri zlib stringio )
libs << File.join(File.dirname(__FILE__), '..', 'bugzilla')
libs << File.join(File.dirname(__FILE__), '..', 'glsamaker')
libs << File.join(File.dirname(__FILE__), 'utils')
@@ -18,7 +18,7 @@ libs.each { |lib| require lib }
TMPDIR = File.join(File.dirname(__FILE__), '..', '..', 'tmp')
# What year are the first CVEs from?
YEAR = (ENV['START_YEAR'] || 2004).to_i
-BASEURL = 'https://nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%s.xml.gz'
+BASEURL = 'https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-%s.json.gz'
DEBUG = ENV.key? 'DEBUG'
VERBOSE = (ENV.key?('VERBOSE') || DEBUG)
@@ -27,6 +27,23 @@ QUIET = ENV.key? 'QUIET'
fail "I can't be quiet and verbose at the same time..." if QUIET && VERBOSE
namespace :cve do
+
+ desc 'Destroy all CVEs and CPEs'
+ task destroy_all: [:environment, 'db:load_config'] do
+ CveReference.destroy_all
+ Cve.destroy_all
+ Cpe.destroy_all
+ end
+
+
+ desc 'Print all CVEs'
+ task print_all: [:environment, 'db:load_config'] do
+ Cve.find_each do |cve|
+ puts cve.cve_id + ",\"" + cve.summary + "\"," + (cve.cvss || "") + "," + cve.state + "," + cve.published_at.to_formatted_s(:iso8601) + "," + cve.last_changed_at.to_formatted_s(:iso8601) + "," + cve.created_at.to_formatted_s(:iso8601) + "," + cve.updated_at.to_formatted_s(:iso8601)
+ end
+ end
+
+
desc 'Full CVE data import'
task full_import: [:environment, 'db:load_config'] do
start_ts = Time.now
@@ -34,43 +51,16 @@ namespace :cve do
(YEAR..Date.today.year).each do |year|
info 'Processing CVEs from '.bold + year.to_s.purple
- xmldata = status 'Downloading' do
+ jsondata = status 'Downloading' do
gunzip_str Glsamaker::HTTP.get(BASEURL % year)
end
- xml = status 'Loading XML' do
- Nokogiri::XML(xmldata)
+ json = status 'Loading JSON' do
+ JSON.parse(jsondata)
end
- namespace = { 'cve' => 'http://scap.nist.gov/schema/feed/vulnerability/2.0' }
- processed_cves = 0
- cpe_cache = {}
-
- cves = xml.root.xpath('cve:entry', namespace)
- info "#{cves.size.to_s.purple} CVEs (one dot equals 100, purple dot equals 500)"
-
- cves.each do |cve|
- unless VERBOSE || QUIET
-
- if processed_cves % 500 == 0
- print '.'.purple
- else
- print '.' if processed_cves % 100 == 0
- end
- end
-
- puts cve['id'].to_s.purple if VERBOSE
-
- begin
- create_cve(cve)
- rescue ActiveRecord::StatementInvalid => e
- # Ignore dupes, ONLY do that for the full db update!
- raise e unless e.message =~ /Duplicate entry/
- end
-
- processed_cves += 1
- STDOUT.flush
- end
+ cves = json["CVE_Items"]
+ create_cves(cves)
puts
end
@@ -79,139 +69,273 @@ namespace :cve do
info "(#{Time.now - start_ts} seconds)"
end
- desc 'Incremental CVE data update'
+
+ desc 'Incrementally update last CVEs'
task update: :environment do
- start_ts = Time.now
+
info 'Running incremental CVE data update...'
- xmldata = status 'Downloading' do
+ jsondata = status 'Downloading' do
gunzip_str Glsamaker::HTTP.get(BASEURL % 'modified')
end
- xml = status 'Loading XML' do
- Nokogiri::XML(xmldata)
+ json = status 'Loading JSON' do
+ JSON.parse(jsondata)
end
- namespace = { 'cve' => 'http://scap.nist.gov/schema/feed/vulnerability/2.0' }
- processed_cves = created_cves = updated_cves = 0
- cpe_cache = {}
+ cves = json["CVE_Items"]
+ update_cves(cves)
+ end
+
+
+ desc 'Update all CVEs'
+ task update_all: :environment do
- cves = xml.root.xpath('cve:entry', namespace)
- info "#{cves.size.to_s.purple} CVEs (one dot equals 100, purple dot equals 500)"
+ info 'Running complete CVE data update...'
+
+ (YEAR..Date.today.year).each do |year|
+ info 'Processing CVEs from '.bold + year.to_s.purple
+
+ jsondata = status 'Downloading' do
+ gunzip_str Glsamaker::HTTP.get(BASEURL % year)
+ end
+
+ json = status 'Loading JSON' do
+ JSON.parse(jsondata)
+ end
- cves.each do |cve|
- unless VERBOSE || QUIET
- if processed_cves % 500 == 0
- print '.'.purple
- else
- print '.' if processed_cves % 100 == 0
+ cves = json["CVE_Items"]
+ update_cves(cves)
+ end
+ end
+
+end
+
+
+#
+# Update the given cve
+#
+def create_cve(cve)
+ summary = cve.dig('cve', 'description', 'description_data', 0, 'value')
+ _cve = Cve.create(
+ :cve_id => cve.dig('cve', 'CVE_data_meta', 'ID'),
+ :summary => summary,
+ :cvss => cve.dig('impact', 'baseMetricV2', 'cvssV2', 'vectorString'),
+ :published_at => DateTime.parse(cve['publishedDate']),
+ :last_changed_at => DateTime.parse(cve['lastModifiedDate']),
+ :state => (summary =~ /^\*\* REJECT \*\*/ ? 'REJECTED' : 'NEW')
+ )
+
+ if cve.dig('references', 'reference_data')
+ cve.dig('references', 'reference_data').each do |ref|
+ CveReference.create(
+ :cve => _cve,
+ :source => ref['refsource'],
+ :title => ref['name'],
+ :uri => ref['url']
+ )
+ end
+ end
+
+ cve.dig('configurations', 'nodes').each do |node|
+ if node.key?('cpe_match')
+ node['cpe_match'].each do |cpe_match|
+ cpe_str = cpe_match['cpe23Uri']
+
+ cpe = Cpe.where(:cpe => cpe_str).first
+ cpe ||= Cpe.create(:cpe => cpe_str)
+
+ _cve.cpes << cpe
+ end
+ elsif node.key?('children')
+ node['children'].each do |child|
+ if child.key?('cpe_match')
+ child['cpe_match'].each do |cpe_match|
+ cpe_str = cpe_match['cpe23Uri']
+
+ cpe = Cpe.where(:cpe => cpe_str).first
+ cpe ||= Cpe.create(:cpe => cpe_str)
+
+ _cve.cpes << cpe
+ end
end
end
+ end
+ end
+end
- puts cve['id'].to_s.purple if VERBOSE
- c = Cve.find_by_cve_id cve['id']
+#
+# Create the all given cves
+#
+def create_cves(cves)
+ processed_cves = 0
+ info "#{cves.size.to_s.purple} CVEs (one dot equals 100, purple dot equals 500)"
+
+ cves.each do |cve|
+ unless VERBOSE || QUIET
- if c.nil?
- debug 'Creating CVE.'
- create_cve(cve)
- created_cves += 1
+ if processed_cves % 500 == 0
+ print '.'.purple
else
- last_changed_at = Time.parse(cve.xpath('vuln:last-modified-datetime').first.content).utc
- db_lca = c.last_changed_at
-
- if last_changed_at.to_i > c.last_changed_at.to_i
- debug 'Updating CVE. Timestamp changed.'
- summary = cve.xpath('vuln:summary').first.content
- c.attributes = {
- cve_id: cve['id'],
+ print '.' if processed_cves % 100 == 0
+ end
+ end
+
+ puts cve.dig('cve', 'CVE_data_meta', 'ID').to_s.purple if VERBOSE
+
+ begin
+ create_cve(cve)
+ rescue ActiveRecord::StatementInvalid => e
+ # Ignore dupes, ONLY do that for the full db update!
+ raise e unless e.message =~ /Duplicate entry/
+ end
+
+ processed_cves += 1
+ STDOUT.flush
+ end
+end
+
+
+#
+# Update the all given cves
+#
+def update_cves(cves)
+ start_ts = Time.now
+ processed_cves = created_cves = updated_cves = 0
+
+ info "#{cves.size.to_s.purple} CVEs (one dot equals 100, purple dot equals 500)"
+
+ cves.each do |cve|
+ unless VERBOSE || QUIET
+ if processed_cves % 500 == 0
+ print '.'.purple
+ else
+ print '.' if processed_cves % 100 == 0
+ end
+ end
+
+ puts cve.dig('cve', 'CVE_data_meta', 'ID').to_s.purple if VERBOSE
+
+ c = Cve.find_by_cve_id cve.dig('cve', 'CVE_data_meta', 'ID')
+
+ if c.nil?
+ debug 'Creating CVE.'
+ create_cve(cve)
+ created_cves += 1
+ else
+ last_changed_at = Time.parse(cve['lastModifiedDate']).utc
+
+ if last_changed_at.to_i > c.last_changed_at.to_i
+ debug 'Updating CVE. Timestamp changed.'
+ summary = cve.dig('cve', 'description', 'description_data', 0, 'value')
+ c.attributes = {
+ cve_id: cve.dig('cve','CVE_data_meta','ID'),
summary: summary,
- cvss: cvss_xml2str(cve.xpath('vuln:cvss')),
- published_at: DateTime.parse(cve.xpath('vuln:published-datetime').first.content),
- last_changed_at: DateTime.parse(cve.xpath('vuln:last-modified-datetime').first.content)
- }
+ cvss: cve.dig('impact', 'baseMetricV2', 'cvssV2', 'vectorString'),
+ published_at: DateTime.parse(cve['publishedDate']),
+ last_changed_at: DateTime.parse(cve['lastModifiedDate']),
+ }
- c.state = 'REJECTED' if summary =~ /^\*\* REJECT \*\*/
- c.save!
+ c.state = 'REJECTED' if summary =~ /^\*\* REJECT \*\*/
+ c.save!
- db_references = []
- xml_references = []
- cve.xpath('vuln:references').each do |ref|
+ db_references = []
+ xml_references = []
+
+ if cve.dig 'references', 'reference_data'
+ cve.dig('references', 'reference_data').each do |ref|
xml_references << [
- ref.xpath('vuln:source').first.content,
- ref.xpath('vuln:reference').first.content,
- ref.xpath('vuln:reference').first['href']
+ :source => ref['refsource'],
+ :title => ref['name'],
+ :uri => ref['url']
]
end
+ end
- c.references.each do |ref|
- db_references << [ref.source, ref.title, ref.uri]
- end
+ c.references.each do |ref|
+ db_references << [ref.source, ref.title, ref.uri]
+ end
- rem = db_references - xml_references
- debug "Removing references: #{rem.inspect}"
+ rem = db_references - xml_references
+ debug "Removing references: #{rem.inspect}"
- rem.each do |item|
- ref = c.references.where(['source = ? AND title = ? AND uri = ?', *item]).first
- debug ref
- c.references.delete(ref)
- ref.destroy
- end
+ rem.each do |item|
+ ref = c.references.where(['source = ? AND title = ? AND uri = ?', *item]).first
+ debug ref
+ c.references.delete(ref)
+ ref.destroy
+ end
- add = xml_references - db_references
- debug "Ading references: #{add.inspect}"
+ add = xml_references - db_references
+ debug "Ading references: #{add.inspect}"
- add.each do |item|
- c.references.create(
+ add.each do |item|
+ c.references.create(
source: item[0],
title: item[1],
uri: item[2]
- )
- end
-
- db_cpes = []
- xml_cpes = []
- cve.xpath('vuln:vulnerable-software-list/vuln:product').each do |prod|
- xml_cpes << prod.content
- end
+ )
+ end
- c.cpes.each do |prod|
- db_cpes << prod.cpe
+ db_cpes = []
+ xml_cpes = []
+
+ cve.dig('configurations', 'nodes').each do |node|
+ if node.key?('cpe_match')
+ node['cpe_match'].each do |cpe_match|
+ xml_cpes << cpe_match['cpe23Uri']
+ end
+ elsif node.key?('children')
+ node['children'].each do |child|
+ if child.key?('cpe_match')
+ child['cpe_match'].each do |cpe_match|
+ xml_cpes << cpe_match['cpe23Uri']
+ end
+ end
+ end
end
+ end
- rem = db_cpes - xml_cpes
- debug "Removing CPEs: #{rem.inspect}"
+ c.cpes.each do |prod|
+ db_cpes << prod.cpe
+ end
- rem.each do |item|
- c.cpes.delete(Cpe.find_by_cpe(item))
- end
+ rem = db_cpes - xml_cpes
+ debug "Removing CPEs: #{rem.inspect}"
- add = xml_cpes - db_cpes
- debug "Ading CPEs: #{add.inspect}"
+ rem.each do |item|
+ c.cpes.delete(Cpe.find_by_cpe(item))
+ end
- add.each do |item|
- cpe = Cpe.where(cpe: item).first
- cpe ||= Cpe.create(cpe: item)
+ add = xml_cpes - db_cpes
+ debug "Ading CPEs: #{add.inspect}"
- c.cpes << cpe
- end
+ add.each do |item|
+ cpe = Cpe.where(cpe: item).first
+ cpe ||= Cpe.create(cpe: item)
- c.save!
- updated_cves += 1
+ c.cpes << cpe
end
- end
- processed_cves += 1
- STDOUT.flush
+ c.save!
+ updated_cves += 1
+ end
end
- info ''
-
- info "(#{Time.now - start_ts} seconds, #{created_cves} new CVE entries, #{updated_cves} updated CVE entries)"
+ processed_cves += 1
+ STDOUT.flush
end
+
+ info ''
+
+ info "(#{Time.now - start_ts} seconds, #{created_cves} new CVE entries, #{updated_cves} updated CVE entries)"
end
+
+#
# Run something, and display a status info around it
+#
def status(message)
fail ArgumentError, 'I want a block :(' unless block_given?
@@ -223,63 +347,19 @@ def status(message)
stuff
end
+
def debug(msg)
$stderr.puts msg if DEBUG
end
+
def info(msg)
puts msg unless QUIET
end
-# 7.2/AV:L/AC:L/Au:N/C:C/I:C/A:C
-def cvss_xml2str(data)
- def get_content(x, y)
- x.xpath(y).first.content
- end
-
- return nil if data.size == 0
-
- str = "#{get_content(data, 'cvss:base_metrics/cvss:score')}/"
- str += "AV:#{get_content(data, 'cvss:base_metrics/cvss:access-vector')[0, 1]}/"
- str += "AC:#{get_content(data, 'cvss:base_metrics/cvss:access-complexity')[0, 1]}/"
- str += "Au:#{get_content(data, 'cvss:base_metrics/cvss:authentication')[0, 1]}/"
- str += "C:#{get_content(data, 'cvss:base_metrics/cvss:confidentiality-impact')[0, 1]}/"
- str += "I:#{get_content(data, 'cvss:base_metrics/cvss:integrity-impact')[0, 1]}/"
- str += "A:#{get_content(data, 'cvss:base_metrics/cvss:availability-impact')[0, 1]}"
-
- str
-end
-
-def create_cve(cve)
- summary = cve.xpath('vuln:summary').first.content
- _cve = Cve.create(
- :cve_id => cve['id'],
- :summary => summary,
- :cvss => cvss_xml2str(cve.xpath('vuln:cvss')),
- :published_at => DateTime.parse(cve.xpath('vuln:published-datetime').first.content),
- :last_changed_at => DateTime.parse(cve.xpath('vuln:last-modified-datetime').first.content),
- :state => (summary =~ /^\*\* REJECT \*\*/ ? 'REJECTED' : 'NEW')
- )
-
- cve.xpath('vuln:references').each do |ref|
- CveReference.create(
- :cve => _cve,
- :source => ref.xpath('vuln:source').first.content,
- :title => ref.xpath('vuln:reference').first.content,
- :uri => ref.xpath('vuln:reference').first['href']
- )
- end
-
- cve.xpath('vuln:vulnerable-software-list/vuln:product').each do |prod|
- cpe_str = prod.content
-
- cpe = Cpe.where(:cpe => cpe_str).first
- cpe ||= Cpe.create(:cpe => cpe_str)
-
- _cve.cpes << cpe
- end
-end
-
+#
+# unzip the given string
+#
def gunzip_str(str)
Zlib::GzipReader.new(StringIO.new(str)).read
end