20110328-setting-renewal_flag-oddb_csv_export-testcases-state-admin-root_rb

<< | Index | >>

1. Check source code
2. Debug wrong arranged columns
3. Update testcases
4. Continue to update testcases

Goal/Estimate/Evaluation

• setting the renewal_flag in oddb_csv export / 80% / 100%

Milestones

1. Look for the part of exporting oddb.csv
2. Confirm the process of exporting
3. Update / BraSt 11:00
4. Update test-cases 13:30
5. src/state/admin/root.rb (coverage: 52.30%)

Summary

• http://masay.ch/2011/03/28/setting-renewal_flag-oddb_csv_export-testcases-state-admin-root_rb/

Commits

• Added the renewal_flag column in exported oddb.csv file (oddb.org)
• Updated testcases, test_state/admin/root.rb (oddb.org)

Check source code

Email

• Task for Monday: Setting the :renewal_flag in oddb.csv Export

Browser

• Drug search -> click Registration number -> check box: 'In renewal'

Source data

• www.swissmedic.ch -> Heilmitteldaten -> Zugelassene Präparate, Verfahren und Wirkstoffe -> Human- und Tierarzneimittel -> Excel-Version Zugelassene Verpackungen
• http://www.swissmedic.ch/daten/00080/00251/index.html?lang=de&download=NHzLpZeg7t,lnp6I0NTU042l2Z6ln1acy4Zn4Z2qZpnO2Yuq2Z6gpJCDdH56fWym162epYbg2c_JjKbNoKSn6A--

Source code

src/util/updater.rb#export_oddb_csv

    def export_oddb_csv(date = @@today)
      subj = 'oddb.csv'
      wrap_update(CsvExportPlugin, subj) {
        plug = CsvExportPlugin.new(@app)
        plug.export_drugs
        log = Log.new(date)
        log.update_values(log_info(plug))
        log.notify(subj)
      }
    end
    def export_oddb2_csv(date = @@today)
      subj = 'oddb2.csv'
      wrap_update(CsvExportPlugin, subj) {
        plug = CsvExportPlugin.new(@app)
        plug.export_drugs_extended
        log = Log.new(date)
        log.update_values(log_info(plug))
        log.notify(subj)
      }
    end

src/util/exporter.rb#export_csv

    def export_csv
      plug = CsvExportPlugin.new(@app)
      safe_export 'oddb.csv' do
        plug.export_drugs
      end
      safe_export 'oddb.csv' do
        plug.export_drugs_extended
      end
      EXPORT_SERVER.clear
      sleep(30)
    end

Note

• Both above actually call the CsvExportPlugin#export_drugs method

src/plugin/csv_export.rb#export_drugs

    def export_drugs
      @options = { :iconv => 'ISO-8859-1//TRANSLIT//IGNORE' }
      recipients.concat self.class::ODDB_RECIPIENTS
      _export_drugs 'oddb', [ :rectype, :iksnr, :ikscd, :ikskey, :barcode,
        :bsv_dossier, :pharmacode, :name_base, :galenic_form,
        :most_precise_dose, :size, :numerical_size, :price_exfactory,
        :price_public, :company_name, :ikscat, :sl_entry, :introduction_date,
        :limitation, :limitation_points, :limitation_text, :lppv,
        :registration_date, :expiration_date, :inactive_date, :export_flag,
        :casrn, :generic_type, :has_generic, :deductible, :out_of_trade,
        :c_type, :index_therapeuticus, :ith_swissmedic, :narcotic, :vaccine ]
    end

And this above called _export_drugs method below

src/plugin/csv_export.rb#_export_drugs

    def _export_drugs(export_name, keys)
      session = SessionStub.new(@app)
      session.language = 'de'
      session.lookandfeel = LookandfeelBase.new(session)
      model = @app.atc_classes.values.sort_by { |atc| atc.code }
      name = "#{export_name}.csv"
      @file_path = path = File.join(EXPORT_DIR, name)
      exporter = View::Drugs::CsvResult.new(model, session)
      exporter.to_csv_file(keys, path, :packages)
      dups = exporter.duplicates
      @counts = exporter.counts
      @counts['duplicates'] = dups.size
      unless(dups.empty?)
        log = Log.new(@@today)
        log.report = sprintf "CSV-Export includes %i duplicates:\n%s",
                             dups.size, dups.join("\n")
        log.notify("CSV-Export includes %i duplicates" % dups.size)
      end
      EXPORT_SERVER.compress(EXPORT_DIR, name)
      backup = @app.log_group(:bsv_sl).newest_date.strftime("#{export_name}.%Y-%m-%d.csv")
      backup_dir = File.expand_path('../../data/csv', File.dirname(__FILE__))
      backup_path = File.join(backup_dir, backup)
      unless(File.exist? backup_path)
        FileUtils.mkdir_p(backup_dir)
        FileUtils.cp(path, backup_path)
      end
    rescue
      puts $!.message
      puts $!.backtrace
      raise
    end

Note

• The renewal_flag is set in src/plugin/swissmedic.rb

src/plugin/swissmedic.rb#update_registration

    def update_registration(row, opts = {})
      opts = {:date => @@today, :create_only => false}.update(opts)
      opts[:date] ||= @@today
      group = cell(row, column(:production_science))
      if(group != 'Tierarzneimittel')
        iksnr = cell(row, column(:iksnr))
        return if (filter = opts[:iksnr]) && iksnr != filter
        return if (filter = opts[:iksnrs]) && !filter.include?(iksnr)
        science = cell(row, column(:production_science))
        ptr = if(registration = @app.registration(iksnr))
                return registration if opts[:create_only]
                registration.pointer
              else
                Persistence::Pointer.new([:registration, iksnr]).creator
              end
        expiration = date_cell(row, column(:expiry_date))
        reg_date = date_cell(row, column(:registration_date))
        args = {
          :ith_swissmedic      => cell(row, column(:index_therapeuticus)),
          :production_science  => science,
          :registration_date   => reg_date,
          :expiration_date     => expiration,
          :renewal_flag        => false,
          :inactive_date       => nil,
          :export_flag         => nil,
        }
        if(expiration < opts[:date])
          args.store :renewal_flag, true
        end
        case science
        when "Anthroposophika"
          args.store :complementary_type, 'anthroposophy'
        when "Homöopathika"
          args.store :complementary_type, 'homeopathy'
        when "Phytotherapeutika"
          args.store :complementary_type, 'phytotherapy'
        end
        if(company = update_company(row))
          args.store :company, company.pointer
        end
        if(indication = update_indication(cell(row, column(:indication_registration))))
          args.store :indication, indication.pointer
        end
        @app.update ptr, args, :swissmedic
      end

Question

• Which object (class) does this renewal_flag blong to?

Test output oddb.csv (default)

• Run
  1. oddb.org/bin/odddbd
  2. currencyd
  3. oddb.org/ext/export/bin/exportd

$ bin/admin 
ch.oddb> Updater.new(self).export_oddb_csv

Result

• Email ch.ODDB.org Report - oddb.csv - 03/2011
• oddb.csv file is added

anthroposophy                      294
bsv_dossiers                      3312
complementary                      282
deductible_g                      8617
deductible_o                        46
duplicates                           0
expiration_date                  21933
export_registrations               668
galenic_forms                      324
generics                          4625
has_generic                          0
homeopathy                         810
inactive_date                     4332
limitation_both                    142
limitation_points                  142
limitation_texts                  1315
limitations                       1315
lppv                               601
missing_size                        28
originals                         2110
out_of_trade                      3447
phytotherapy                      1342
price_exfactory                   9833
price_public                     12369
registration_date                23566
routes_of_administration             0
sl_entries                        8663

Experiment

src/plugin/csv_export.rb#export_drugs

    def export_drugs
      @options = { :iconv => 'ISO-8859-1//TRANSLIT//IGNORE' }
      recipients.concat self.class::ODDB_RECIPIENTS
      _export_drugs 'oddb', [ :rectype, :iksnr, :ikscd, :ikskey, :barcode,
        :bsv_dossier, :pharmacode, :name_base, :galenic_form,
        :most_precise_dose, :size, :numerical_size, :price_exfactory,
        :price_public, :company_name, :ikscat, :sl_entry, :introduction_date,
        :limitation, :limitation_points, :limitation_text, :lppv,
        :registration_date, :expiration_date, :inactive_date, :export_flag,
        :casrn, :generic_type, :has_generic, :deductible, :out_of_trade,
        #:c_type, :index_therapeuticus, :ith_swissmedic, :narcotic, :vaccine ]
        :c_type, :index_therapeuticus, :ith_swissmedic, :narcotic, :vaccine, :renewal_flag ]
    end

Rerun

1. oddb.org/bin/odddbd
2. currencyd
3. oddb.org/ext/export/bin/exportd

$ bin/admin 
ch.oddb> Updater.new(self).export_oddb_csv

Result

• Email ch.ODDB.org Report - Error: oddb.csv - 03/2011 Error

Plugin: ODDB::CsvExportPlugin
Error: NoMethodError
Message: undefined method `renewal_flag' for #<ODDB::Package:0x7f9ce33e7c68>
Backtrace:
(eval):2:in `send'
(eval):2:in `send'
/home/masa/ywesee/oddb.org/src/view/drugs/csv_result.rb:323:in `to_csv'
/home/masa/ywesee/oddb.org/src/view/drugs/csv_result.rb:319:in `collect'
/home/masa/ywesee/oddb.org/src/view/drugs/csv_result.rb:319:in `to_csv'
/home/masa/ywesee/oddb.org/src/view/drugs/csv_result.rb:317:in `each'
/home/masa/ywesee/oddb.org/src/view/drugs/csv_result.rb:317:in `to_csv'
/home/masa/ywesee/oddb.org/src/view/drugs/csv_result.rb:315:in `each'
/home/masa/ywesee/oddb.org/src/view/drugs/csv_result.rb:315:in `to_csv'
/home/masa/ywesee/oddb.org/src/view/drugs/csv_result.rb:336:in `to_csv_file'
/home/masa/ywesee/oddb.org/src/view/drugs/csv_result.rb:336:in `open'
/home/masa/ywesee/oddb.org/src/view/drugs/csv_result.rb:336:in `to_csv_file'
/home/masa/ywesee/oddb.org/src/plugin/csv_export.rb:55:in `_export_drugs'
/home/masa/ywesee/oddb.org/src/plugin/csv_export.rb:24:in `export_drugs'
/home/masa/ywesee/oddb.org/src/util/updater.rb:81:in `export_oddb_csv'
/home/masa/ywesee/oddb.org/src/util/updater.rb:461:in `call'
/home/masa/ywesee/oddb.org/src/util/updater.rb:461:in `wrap_update'
/home/masa/ywesee/oddb.org/src/util/updater.rb:79:in `export_oddb_csv'
(eval):1:in `_admin'
/home/masa/ywesee/oddb.org/src/util/failsafe.rb:9:in `call'
/home/masa/ywesee/oddb.org/src/util/failsafe.rb:9:in `failsafe'
/home/masa/ywesee/oddb.org/src/util/oddbapp.rb:1399:in `_admin'
/home/masa/ywesee/oddb.org/src/util/oddbapp.rb:1397:in `initialize'
/home/masa/ywesee/oddb.org/src/util/oddbapp.rb:1397:in `new'
/home/masa/ywesee/oddb.org/src/util/oddbapp.rb:1397:in `_admin'
/usr/lib64/ruby/1.8/drb/drb.rb:1555:in `__send__'
/usr/lib64/ruby/1.8/drb/drb.rb:1555:in `perform_without_block'
/usr/lib64/ruby/1.8/drb/drb.rb:1515:in `perform'
/usr/lib64/ruby/1.8/drb/drb.rb:1589:in `main_loop'
/usr/lib64/ruby/1.8/drb/drb.rb:1585:in `loop'
/usr/lib64/ruby/1.8/drb/drb.rb:1585:in `main_loop'
/usr/lib64/ruby/1.8/drb/drb.rb:1581:in `start'
/usr/lib64/ruby/1.8/drb/drb.rb:1581:in `main_loop'
/usr/lib64/ruby/1.8/drb/drb.rb:1430:in `run'
/usr/lib64/ruby/1.8/drb/drb.rb:1427:in `start'
/usr/lib64/ruby/1.8/drb/drb.rb:1427:in `run'
/usr/lib64/ruby/1.8/drb/drb.rb:1347:in `initialize'
/usr/lib64/ruby/1.8/drb/drb.rb:1627:in `new'
/usr/lib64/ruby/1.8/drb/drb.rb:1627:in `start_service'
bin/oddbd:38

Experiment

src/model/package.rb

      #:source, :index_therapeuticus, :ith_swissmedic, :has_fachinfo?, :production_science
      :source, :index_therapeuticus, :ith_swissmedic, :has_fachinfo?, :production_science, :renewal_flag

Rerun

1. oddb.org/bin/odddbd
2. currencyd
3. oddb.org/ext/export/bin/exportd

$ bin/admin 
ch.oddb> Updater.new(self).export_oddb_csv

Result

• Email ch.ODDB.org Report - oddb.csv - 03/2011
• The last column of added oddb.csv as follows

Note

• This task has almost done.
• Next:
  1. Check columns order
  2. Check test-cases

Debug wrong arranged columns

Problem

• Several columns in oddb.csv are arranged wrong

Example

• Registration number: 58255, 60543

Note

• These records have both 'description' and 'galenic form' columns
• The 'description' data is inserted in the 'galenic form' column, and
• the 'galenic form' data is inserted in 'Stärke' column

Resolved

• We should open oddb.csv with semicolon without comma

Update testcases

Check all the test-cases

  1) Failure:
test_recent_items(ODDB::TestPatinfoInvoicer) [/home/masa/ywesee/oddb.org/test/test_plugin/patinfo_invoicer.rb:183]:
<2> expected but was
<1>.

  2) Error:
test_download_export_descr(TestDownloadExportComposite):
NoMethodError: undefined method `user_input' for <FlexMock:session>:FlexMock
    /usr/lib64/ruby/gems/1.8/gems/flexmock-0.8.6/lib/flexmock/core.rb:105:in `method_missing'
    /usr/lib64/ruby/gems/1.8/gems/flexmock-0.8.6/lib/flexmock/core.rb:191:in `flexmock_wrap'
    /usr/lib64/ruby/gems/1.8/gems/flexmock-0.8.6/lib/flexmock/core.rb:98:in `method_missing'
    /home/masa/ywesee/oddb.org/src/view/user/export.rb:84:in `once_or_year'
    /home/masa/ywesee/oddb.org/src/view/user/download_export.rb:420:in `radio_oddb_csv'
    /usr/lib64/ruby/site_ruby/1.8/htmlgrid/composite.rb:66:in `send'
    /usr/lib64/ruby/site_ruby/1.8/htmlgrid/composite.rb:66:in `create'
    /usr/lib64/ruby/site_ruby/1.8/htmlgrid/composite.rb:280:in `compose_component'
    /usr/lib64/ruby/site_ruby/1.8/htmlgrid/composite.rb:209:in `compose'
    /usr/lib64/ruby/site_ruby/1.8/htmlgrid/composite.rb:203:in `each'
    /usr/lib64/ruby/site_ruby/1.8/htmlgrid/composite.rb:203:in `compose'
    /usr/lib64/ruby/site_ruby/1.8/htmlgrid/composite.rb:55:in `init'
    /usr/lib64/ruby/site_ruby/1.8/htmlgrid/component.rb:138:in `initialize'
    /usr/lib64/ruby/site_ruby/1.8/htmlgrid/composite.rb:59:in `new'
    /usr/lib64/ruby/site_ruby/1.8/htmlgrid/composite.rb:59:in `create'
    /usr/lib64/ruby/site_ruby/1.8/htmlgrid/composite.rb:280:in `compose_component'
    /usr/lib64/ruby/site_ruby/1.8/htmlgrid/composite.rb:209:in `compose'
    /usr/lib64/ruby/site_ruby/1.8/htmlgrid/composite.rb:203:in `each'
    /usr/lib64/ruby/site_ruby/1.8/htmlgrid/composite.rb:203:in `compose'
    /usr/lib64/ruby/site_ruby/1.8/htmlgrid/composite.rb:55:in `init'
    /usr/lib64/ruby/site_ruby/1.8/htmlgrid/form.rb:82:in `init'
    /home/masa/ywesee/oddb.org/src/view/user/download_export.rb:535:in `init'
    /usr/lib64/ruby/site_ruby/1.8/htmlgrid/component.rb:138:in `initialize'
    /home/masa/ywesee/oddb.org/test/test_view/user/download_export.rb:51:in `new'
    /home/masa/ywesee/oddb.org/test/test_view/user/download_export.rb:51:in `setup'

  3) Error:
test_compression_label(TestDownloadExportInnerComposite):
NoMethodError: undefined method `user_input' for <FlexMock:session>:FlexMock
    /usr/lib64/ruby/gems/1.8/gems/flexmock-0.8.6/lib/flexmock/core.rb:105:in `method_missing'
    /usr/lib64/ruby/gems/1.8/gems/flexmock-0.8.6/lib/flexmock/core.rb:191:in `flexmock_wrap'
    /usr/lib64/ruby/gems/1.8/gems/flexmock-0.8.6/lib/flexmock/core.rb:98:in `method_missing'
    /home/masa/ywesee/oddb.org/src/view/user/export.rb:84:in `once_or_year'
    /home/masa/ywesee/oddb.org/src/view/user/download_export.rb:420:in `radio_oddb_csv'
    /usr/lib64/ruby/site_ruby/1.8/htmlgrid/composite.rb:66:in `send'
    /usr/lib64/ruby/site_ruby/1.8/htmlgrid/composite.rb:66:in `create'
    /usr/lib64/ruby/site_ruby/1.8/htmlgrid/composite.rb:280:in `compose_component'
    /usr/lib64/ruby/site_ruby/1.8/htmlgrid/composite.rb:209:in `compose'
    /usr/lib64/ruby/site_ruby/1.8/htmlgrid/composite.rb:203:in `each'
    /usr/lib64/ruby/site_ruby/1.8/htmlgrid/composite.rb:203:in `compose'
    /usr/lib64/ruby/site_ruby/1.8/htmlgrid/composite.rb:55:in `init'
    /usr/lib64/ruby/site_ruby/1.8/htmlgrid/component.rb:138:in `initialize'
    /home/masa/ywesee/oddb.org/test/test_view/user/download_export.rb:27:in `new'
    /home/masa/ywesee/oddb.org/test/test_view/user/download_export.rb:27:in `setup'

Note

• 1 Failure, 2 Errors

Update

Git pull and Reboot on server

Commit

• Added the renewal_flag column in exported oddb.csv file (oddb.org)

Continue to update testcases

Updated

• src/state/admin/root.rb (coverage: 100%)

Commit

• Updated testcases, test_state/admin/root.rb (oddb.org)