Class: Argos::Download

Inherits:
Object
  • Object
show all
Defined in:
lib/argos/download.rb

Class Method Summary (collapse)

Class Method Details

+ (Object) download(username, password, archive, log, days = 20)



23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
# File 'lib/argos/download.rb', line 23

def self.download(username, password, archive, log, days=20)
  
  log.debug "Starting download of Argos XML to #{archive}"
  
  soap = Argos::Soap.new({username: username, password: password})
  soap.log = log
  
  programs = soap.programs
  log.debug "Programs for #{username}: #{programs.to_json}"
  year = DateTime.now.year
  
  soap.getPlatformList["data"]["program"].each do |program|
    programNumber = program["programNumber"]
    soap.programNumber = programNumber
  
    platforms = soap.platforms
    
    # inactive (no last collect)
    inactive = program["platform"]
    
    active = program["platform"].select {|platform|
      platform.key? "lastCollectDate" and platform.key? "lastLocationDate"}.select {|platform|
      
      lastCollectDate = DateTime.parse(platform["lastCollectDate"])
      lastLocationDate = DateTime.parse(platform["lastLocationDate"])
      
      twentydays = DateTime.parse((Date.today-20).to_s)
      
      (lastCollectDate > twentydays or lastLocationDate > twentydays)
      
    }
    # inactive = platforms where last collect date is missing or > 20 days
    inactive = program["platform"] - active

    active.each_with_index do |a,m|
      log.debug "Active [#{m+1}/#{active.size}]: #{a.reject{|k,v| k =~ /location/i }.to_json}"
    end
    inactive.each_with_index do |i,n|
      log.debug "Inactive [#{n+1}/#{inactive.size}]: #{i.reject{|k,v| k =~ /location/i }.to_json}"
    end
    active.each_with_index do | platform, idx |
      
  
      platformId = platform["platformId"]
      soap.platformId = platformId
      
      log.debug "About to download program: #{programNumber}, platform: #{platformId} [#{idx+1}/#{active.size}], lastCollectDate: #{platform["lastCollectDate"]}"
    
      20.downto(1) do |daysago|
        
        date = Date.today-daysago
        
        destination = "#{archive}/#{year}/program-#{programNumber}/platform-#{platformId}"
      
        begin
          soap.period = {startDate: "#{date}T00:00:00Z", endDate: "#{date}T23:59:59.999Z"}
          soap.getXml
          
          FileUtils.mkdir_p(destination)
          filename = destination+"/argos-#{date}-platform-#{platformId}.xml"
          
          if File.exists? filename
            existing_sha1 = Digest::SHA1.file(filename).hexdigest
            existing_errors = Argos::Soap.new.validate(File.read(filename))
            if existing_errors.any?
              log.error "Validation error for existing data #{filename} (#{File.size(filename)} bytes): #{existing_errors.uniq.to_json}"
            end 
            #log.debug "Keeping existing file #{filename} from #{File.mtime(filename)}, fresh data is identical"
          end
          
          new_xml_ng = Nokogiri::XML(soap.xml, nil, nil, Nokogiri::XML::ParseOptions::NOBLANKS | Nokogiri::XML::ParseOptions::NOCDATA | Nokogiri::XML::ParseOptions::STRICT)
          new_xml = new_xml_ng.canonicalize
          new_sha1 = Digest::SHA1.hexdigest(new_xml)
          
          if existing_sha1.nil? or existing_errors.any? or existing_sha1 != new_sha1
          
            if errors = Argos::Soap.new.validate(new_xml_ng).any?
              raise "Failed XML schema validation"
            else
              File.open(filename, "wb") { |file| file.write(new_xml)}
              log.debug "Validated and saved new data: #{filename}"
            end
            
          end
          log.debug "Day -#{daysago}: #{date}: #{new_xml_ng.xpath("//message").size} message(s) (program #{programNumber}, platform #{platformId})"
        rescue Argos::NodataException
          # noop
          log.debug "Day -#{daysago}: #{date}: No data for (program #{programNumber}, platform #{platformId})"
        rescue => e
          log.error e
        end
        
      end
      
      log.debug "Completed download of #{platformId}"
      
    end
    log.debug "Completed download for #{username}"
  end
end