<< Masa.20101025-debug-import_gkv-autorun | 2010 | Masa.20101019-debug-import_gkv >>
suspend
suspend
Experiment (in my local)
de.oddb.org/lib/oddb/export/server.rb
def Server.safe_export(exporter_class, name, *args, &block)
dir = ODDB.config.export_dir or raise "Please configure 'export_dir'"
FileUtils.mkdir_p(dir)
Tempfile.open(name, dir) { |fh|
exporter = exporter_class.new
args.push fh
exporter.export *args
fh.close
newpath = File.join(dir, name)
FileUtils.mv(fh.path, newpath)
FileUtils.chmod(0644, newpath)
compress(dir, name)
if(exporter_class == Export::Xls::ComparisonDeCh)
unless(exporter.error_data.empty?)
message = "\nThe following data was not able to be compared due to NoMethodError:\n"
backtrace_info = "The original backtrace information:\n" + exporter.backtrace_info.join("\n").to_s + "\n"
raise NoMethodError, message + exporter.error_data.join("\n").to_s + "\n\n" + backtrace_info
end
end
}
name
de.oddb.org/lib/oddb/export/xls.rb
class ComparisonDeCh
attr_reader :error_data
attr_reader :backtrace_info
def collect_comparables(drb_uri)
data = []
@error_data = []
@backtrace_info = []
count = 0
DRb::DRbObject.new(nil, drb_uri).remote_each_package { |remote|
count += 1
package = Remote::Drugs::Package.new(drb_uri, remote,
1.0 / currency_rate,
tax_factor)
begin
if count == 1 or count == 2
raise NoMethodError
end
if(package.price(:public) \
&& (comparable = package.local_comparables.select { |pac|
pac.price(:public)
}.sort_by { |pac|
pac.price(:public)
}.first))
data.push [comparable, package]
end
rescue NoMethodError => err
@error_data.push("http://ch.oddb.org/en/gcc/compare/ean13/" + package.code(:ean).value)
@backtrace_info.push("\nhttp://ch.oddb.org/en/gcc/compare/ean13/" + package.code(:ean).value)
@backtrace_info.concat err.backtrace
end
nil # don't return data from the block across drb
}
data
end
Run export_chde_xls
Result Email Thu Oct 21 17:11:00 2010: ODDB::Export::Xls::ComparisonDeCh
Thu Oct 21 17:11:00 2010: ODDB::Export::Xls::ComparisonDeCh#export NoMethodError The following data was not able to be compared due to NoMethodError: http://ch.oddb.org/en/gcc/compare/ean13/7680546700362 http://ch.oddb.org/en/gcc/compare/ean13/7680583510016 The original backtrace information: http://ch.oddb.org/en/gcc/compare/ean13/7680546700362 /home/masa/ywesee/de.oddb.org/lib/oddb/export/xls.rb:39:in `collect_comparables' /usr/lib64/ruby/1.8/drb/drb.rb:1556:in `call' /usr/lib64/ruby/1.8/drb/drb.rb:1556:in `__send__' /usr/lib64/ruby/1.8/drb/drb.rb:1556:in `perform_without_block' /usr/lib64/ruby/1.8/drb/drb.rb:1516:in `perform' /usr/lib64/ruby/1.8/drb/drb.rb:1590:in `main_loop' /usr/lib64/ruby/1.8/drb/drb.rb:1586:in `loop' /usr/lib64/ruby/1.8/drb/drb.rb:1586:in `main_loop' /usr/lib64/ruby/1.8/drb/drb.rb:1582:in `start' /usr/lib64/ruby/1.8/drb/drb.rb:1582:in `main_loop' /usr/lib64/ruby/1.8/drb/drb.rb:1431:in `run' /usr/lib64/ruby/1.8/drb/drb.rb:1428:in `start' /usr/lib64/ruby/1.8/drb/drb.rb:1428:in `run' /usr/lib64/ruby/1.8/drb/drb.rb:1348:in `initialize' /usr/lib64/ruby/1.8/drb/drb.rb:1628:in `new' /usr/lib64/ruby/1.8/drb/drb.rb:1628:in `start_service' /home/masa/ywesee/de.oddb.org/lib/oddb/util/job.rb:13:in `run' jobs/export_chde_xls:16 http://ch.oddb.org/en/gcc/compare/ean13/7680583510016 /home/masa/ywesee/de.oddb.org/lib/oddb/export/xls.rb:39:in `collect_comparables' /usr/lib64/ruby/1.8/drb/drb.rb:1556:in `call' /usr/lib64/ruby/1.8/drb/drb.rb:1556:in `__send__' /usr/lib64/ruby/1.8/drb/drb.rb:1556:in `perform_without_block' /usr/lib64/ruby/1.8/drb/drb.rb:1516:in `perform' /usr/lib64/ruby/1.8/drb/drb.rb:1590:in `main_loop' /usr/lib64/ruby/1.8/drb/drb.rb:1586:in `loop' /usr/lib64/ruby/1.8/drb/drb.rb:1586:in `main_loop' /usr/lib64/ruby/1.8/drb/drb.rb:1582:in `start' /usr/lib64/ruby/1.8/drb/drb.rb:1582:in `main_loop' /usr/lib64/ruby/1.8/drb/drb.rb:1431:in `run' /usr/lib64/ruby/1.8/drb/drb.rb:1428:in `start' /usr/lib64/ruby/1.8/drb/drb.rb:1428:in `run' /usr/lib64/ruby/1.8/drb/drb.rb:1348:in `initialize' /usr/lib64/ruby/1.8/drb/drb.rb:1628:in `new' /usr/lib64/ruby/1.8/drb/drb.rb:1628:in `start_service' /home/masa/ywesee/de.oddb.org/lib/oddb/util/job.rb:13:in `run' jobs/export_chde_xls:16 /home/masa/ywesee/de.oddb.org/lib/oddb/export/server.rb:68:in `safe_export' /usr/lib64/ruby/1.8/tempfile.rb:172:in `open' /home/masa/ywesee/de.oddb.org/lib/oddb/export/server.rb:54:in `safe_export' /home/masa/ywesee/de.oddb.org/lib/oddb/export/server.rb:25:in `export_chde_xls' jobs/export_chde_xls:17 /home/masa/ywesee/de.oddb.org/lib/oddb/util/job.rb:16:in `call' /home/masa/ywesee/de.oddb.org/lib/oddb/util/job.rb:16:in `run' jobs/export_chde_xls:16
Commit Temporary solution for a NoMethodError of export_chde_xls
Result (on production server) Email Fri Oct 22 09:37:40 2010: ODDB::Export::Xls::ComparisonDeCh
Fri Oct 22 09:37:40 2010: ODDB::Export::Xls::ComparisonDeCh#export NoMethodError But The chde.xls file was generated successfully. The following data was not able to be compared due to a NoMethodError: http://ch.oddb.org/en/gcc/compare/ean13/7680527950281 http://ch.oddb.org/en/gcc/compare/ean13/7680584560010 http://ch.oddb.org/en/gcc/compare/ean13/7680584560027 The original backtrace information: http://ch.oddb.org/en/gcc/compare/ean13/7680527950281 (druby://localhost:10013) /usr/lib64/ruby/site_ruby/1.8/odba/stub.rb:112:in `send' (druby://localhost:10013) /usr/lib64/ruby/site_ruby/1.8/odba/stub.rb:112:in `method_missing' (druby://localhost:10013) /var/www/oddb.org/src/model/part.rb:31:in `active_agents' (druby://localhost:10013) /usr/lib64/ruby/site_ruby/1.8/odba/stub.rb:112:in `send' (druby://localhost:10013) /usr/lib64/ruby/site_ruby/1.8/odba/stub.rb:112:in `method_missing' (druby://localhost:10013) /var/www/oddb.org/src/model/package.rb:68:in `active_agents' (druby://localhost:10013) /usr/lib64/ruby/1.8/drb/drb.rb:395:in `inject' (druby://localhost:10013) /var/www/oddb.org/src/model/package.rb:68:in `each' (druby://localhost:10013) /var/www/oddb.org/src/model/package.rb:68:in `inject' (druby://localhost:10013) /var/www/oddb.org/src/model/package.rb:68:in `active_agents' /var/www/de.oddb.org/lib/oddb/remote/drugs/package.rb:27:in `active_agents' /var/www/de.oddb.org/lib/oddb/remote/drugs/package.rb:75:in `local_comparables' /var/www/de.oddb.org/lib/oddb/export/xls.rb:32:in `collect_comparables' /usr/lib64/ruby/1.8/drb/drb.rb:1555:in `call' /usr/lib64/ruby/1.8/drb/drb.rb:1555:in `__send__' /usr/lib64/ruby/1.8/drb/drb.rb:1555:in `perform_without_block' /usr/lib64/ruby/1.8/drb/drb.rb:1515:in `perform' /usr/lib64/ruby/1.8/drb/drb.rb:1589:in `main_loop' /usr/lib64/ruby/1.8/drb/drb.rb:1585:in `loop' /usr/lib64/ruby/1.8/drb/drb.rb:1585:in `main_loop' /usr/lib64/ruby/1.8/drb/drb.rb:1581:in `start' /usr/lib64/ruby/1.8/drb/drb.rb:1581:in `main_loop' /usr/lib64/ruby/1.8/drb/drb.rb:1430:in `run' /usr/lib64/ruby/1.8/drb/drb.rb:1427:in `start' /usr/lib64/ruby/1.8/drb/drb.rb:1427:in `run' /usr/lib64/ruby/1.8/drb/drb.rb:1347:in `initialize' /usr/lib64/ruby/1.8/drb/drb.rb:1627:in `new' /usr/lib64/ruby/1.8/drb/drb.rb:1627:in `start_service' /var/www/de.oddb.org/lib/oddb/util/job.rb:13:in `run' jobs/export_chde_xls:16 http://ch.oddb.org/en/gcc/compare/ean13/7680584560010 (druby://localhost:10013) /usr/lib64/ruby/site_ruby/1.8/odba/stub.rb:112:in `send' (druby://localhost:10013) /usr/lib64/ruby/site_ruby/1.8/odba/stub.rb:112:in `method_missing' (druby://localhost:10013) /var/www/oddb.org/src/model/part.rb:31:in `active_agents' (druby://localhost:10013) /usr/lib64/ruby/site_ruby/1.8/odba/stub.rb:112:in `send' (druby://localhost:10013) /usr/lib64/ruby/site_ruby/1.8/odba/stub.rb:112:in `method_missing' (druby://localhost:10013) /var/www/oddb.org/src/model/package.rb:68:in `active_agents' (druby://localhost:10013) /usr/lib64/ruby/1.8/drb/drb.rb:395:in `inject' (druby://localhost:10013) /usr/lib64/ruby/site_ruby/1.8/odba/stub.rb:112:in `each' (druby://localhost:10013) /usr/lib64/ruby/site_ruby/1.8/odba/stub.rb:112:in `inject' (druby://localhost:10013) /usr/lib64/ruby/site_ruby/1.8/odba/stub.rb:112:in `send' (druby://localhost:10013) /usr/lib64/ruby/site_ruby/1.8/odba/stub.rb:112:in `method_missing' (druby://localhost:10013) /var/www/oddb.org/src/model/package.rb:68:in `active_agents' /var/www/de.oddb.org/lib/oddb/remote/drugs/package.rb:27:in `active_agents' /var/www/de.oddb.org/lib/oddb/remote/drugs/package.rb:75:in `local_comparables' /var/www/de.oddb.org/lib/oddb/export/xls.rb:32:in `collect_comparables' /usr/lib64/ruby/1.8/drb/drb.rb:1555:in `call' /usr/lib64/ruby/1.8/drb/drb.rb:1555:in `__send__' /usr/lib64/ruby/1.8/drb/drb.rb:1555:in `perform_without_block' /usr/lib64/ruby/1.8/drb/drb.rb:1515:in `perform' /usr/lib64/ruby/1.8/drb/drb.rb:1589:in `main_loop' /usr/lib64/ruby/1.8/drb/drb.rb:1585:in `loop' /usr/lib64/ruby/1.8/drb/drb.rb:1585:in `main_loop' /usr/lib64/ruby/1.8/drb/drb.rb:1581:in `start' /usr/lib64/ruby/1.8/drb/drb.rb:1581:in `main_loop' /usr/lib64/ruby/1.8/drb/drb.rb:1430:in `run' /usr/lib64/ruby/1.8/drb/drb.rb:1427:in `start' /usr/lib64/ruby/1.8/drb/drb.rb:1427:in `run' /usr/lib64/ruby/1.8/drb/drb.rb:1347:in `initialize' /usr/lib64/ruby/1.8/drb/drb.rb:1627:in `new' /usr/lib64/ruby/1.8/drb/drb.rb:1627:in `start_service' /var/www/de.oddb.org/lib/oddb/util/job.rb:13:in `run' jobs/export_chde_xls:16 http://ch.oddb.org/en/gcc/compare/ean13/7680584560027 (druby://localhost:10013) /usr/lib64/ruby/site_ruby/1.8/odba/stub.rb:112:in `send' (druby://localhost:10013) /usr/lib64/ruby/site_ruby/1.8/odba/stub.rb:112:in `method_missing' (druby://localhost:10013) /var/www/oddb.org/src/model/part.rb:31:in `active_agents' (druby://localhost:10013) /usr/lib64/ruby/site_ruby/1.8/odba/stub.rb:112:in `send' (druby://localhost:10013) /usr/lib64/ruby/site_ruby/1.8/odba/stub.rb:112:in `method_missing' (druby://localhost:10013) /var/www/oddb.org/src/model/package.rb:68:in `active_agents' (druby://localhost:10013) /usr/lib64/ruby/1.8/drb/drb.rb:395:in `inject' (druby://localhost:10013) /usr/lib64/ruby/site_ruby/1.8/odba/stub.rb:112:in `each' (druby://localhost:10013) /usr/lib64/ruby/site_ruby/1.8/odba/stub.rb:112:in `inject' (druby://localhost:10013) /usr/lib64/ruby/site_ruby/1.8/odba/stub.rb:112:in `send' (druby://localhost:10013) /usr/lib64/ruby/site_ruby/1.8/odba/stub.rb:112:in `method_missing' (druby://localhost:10013) /var/www/oddb.org/src/model/package.rb:68:in `active_agents' /var/www/de.oddb.org/lib/oddb/remote/drugs/package.rb:27:in `active_agents' /var/www/de.oddb.org/lib/oddb/remote/drugs/package.rb:75:in `local_comparables' /var/www/de.oddb.org/lib/oddb/export/xls.rb:32:in `collect_comparables' /usr/lib64/ruby/1.8/drb/drb.rb:1555:in `call' /usr/lib64/ruby/1.8/drb/drb.rb:1555:in `__send__' /usr/lib64/ruby/1.8/drb/drb.rb:1555:in `perform_without_block' /usr/lib64/ruby/1.8/drb/drb.rb:1515:in `perform' /usr/lib64/ruby/1.8/drb/drb.rb:1589:in `main_loop' /usr/lib64/ruby/1.8/drb/drb.rb:1585:in `loop' /usr/lib64/ruby/1.8/drb/drb.rb:1585:in `main_loop' /usr/lib64/ruby/1.8/drb/drb.rb:1581:in `start' /usr/lib64/ruby/1.8/drb/drb.rb:1581:in `main_loop' /usr/lib64/ruby/1.8/drb/drb.rb:1430:in `run' /usr/lib64/ruby/1.8/drb/drb.rb:1427:in `start' /usr/lib64/ruby/1.8/drb/drb.rb:1427:in `run' /usr/lib64/ruby/1.8/drb/drb.rb:1347:in `initialize' /usr/lib64/ruby/1.8/drb/drb.rb:1627:in `new' /usr/lib64/ruby/1.8/drb/drb.rb:1627:in `start_service' /var/www/de.oddb.org/lib/oddb/util/job.rb:13:in `run' jobs/export_chde_xls:16 /var/www/de.oddb.org/lib/oddb/export/server.rb:69:in `safe_export' /usr/lib64/ruby/1.8/tempfile.rb:172:in `open' /var/www/de.oddb.org/lib/oddb/export/server.rb:54:in `safe_export' /var/www/de.oddb.org/lib/oddb/export/server.rb:25:in `export_chde_xls' jobs/export_chde_xls:17 /var/www/de.oddb.org/lib/oddb/util/job.rb:16:in `call' /var/www/de.oddb.org/lib/oddb/util/job.rb:16:in `run' jobs/export_chde_xls:16
Goal adding a prefix for the pharmacode field in xls2odat
We need to enable putting a number like
15
into the colum where the pharmacode is required. If the data XLS contains a number in that column then we need to be able to put the following values into the conf-file:
15A
this would mean that for every number in column A, 15 has to be added as a number so if A=234577 the result of xls2odat would be
15234577
Commit Updated analyzer method. It can concatenate string and cell value
Correct xls2odat: Keeping the 00 from the data file
If a column starts with
002344
and we append the
15
the number hast to be
15002344
not
152344
Notes
Refer
Notes
if value =~ /\d{5}/ # for EAN code and pharmacode (long digit)
value = value.to_i
end
Commit Keeping the 00 from the data file
If tables 6 and 7 are in the conf.xls the please set the file names to s01 etc. Also please do not output any logfiles. If there are no tables 6 and 7 then please output the file name H01 etc. _plus_ the log files.
Commit Change file name depending on odat version
Task stick to the order: xls2odat confg.xls datafile.xls
Davatz-san got the following error because of the missing of the order of flies, configuration.xls and data.xls
/usr/lib/ruby/site_ruby/1.8/xls2odat.rb:47:in `read_config': undefined
method `delete_if' for nil:NilClass (NoMethodError)
from /usr/bin/xls2odat:19
But usually the script prints the following error message if the configuration file is in wrong format
masa@masa ~/work/xls2odat $ ruby -I lib/ bin/xls2odat data.xls conf.xls
./lib/xls2odat.rb:52:in `read_config': Config file format is wrong (Artikel Nr., Bezeichnung, Bezeichnung, EAN-Nummer, Pharma Code, Liefereinheit, Originalgebinde, Einstandspreis, Publikumsspreis, , , , , , , , , , , , , , , , , , , , , , , , , , ) (RuntimeError)
from bin/xls2odat:19
Task Ersatzartikel mit Flag "REP" - table 20, fields 4 and 5 V1.1
If you find the words
"Ersatzartikel für"
in column B then you have to take the the number
002183
and add
15
and put the code
15002183
into table 20 field 4 and put the value
REP
into table 20 field 5
suspend
Task Zubef - jobs/import_gkv does not run automatically
The import_gkv script does not check automatically if there is a new Zubef-PDF file or not. Actually it should do so but it does not do it at the moment. The Updater should check for a new file (I guess every day) and then import if there is a new file.
Check
grep check
masa@masa ~/ywesee/de.oddb.org $ grep -r import_gkv *
jobs/import_gkv: Updater.import_gkv $opts
lib/oddb/util/updater.rb: def Updater.import_gkv(opts = {})
lib/oddb/util/updater.rb: print "WARNING: Updater.import_gkv did nothing. It looks failing in grabbing PDF link.\n"
lib/oddb/util/updater.rb: run_logged_job 'import_gkv'
test/util/test_updater.rb: cmds = %w{import_dimdi import_gkv import_pharmnet import_whocc}
test/util/test_updater.rb: cmds = %w{import_gkv import_pharmnet import_whocc}
test/util/test_updater.rb: def test_import_gkv
test/util/test_updater.rb: @updater.import_gkv
lib/oddb/util/updater.rb Updater.run
def Updater.run(today = Date.today)
run_logged_job 'import_dimdi'
run_logged_job 'import_gkv'
case today.day
when 1
run_logged_job 'import_pharmnet'
run_logged_job 'import_whocc'
when 15
run_logged_job 'import_pharma24'
end
end
Notes
Question
grep
masa@masa ~/ywesee/de.oddb.org $ grep -r Updater.run *
lib/oddb/util/server.rb: @updater = run_at(ODDB.config.update_hour) { Updater.run }
lib/oddb/util/updater.rb: def Updater.run(today = Date.today)
lib/oddb/util/updater.rb: def Updater.run_logged_job job
def run_updater
@updater = run_at(ODDB.config.update_hour) { Updater.run }
end
Notes
grep
masa@masa ~/ywesee/de.oddb.org $ grep -r run_updater * etc/oddb.yml:run_updater: true etc/oddb.yml.bak:run_updater: true lib/oddb/config.rb: 'run_updater' => true, lib/oddb/util/server.rb: run_updater if(ODDB.config.run_updater) lib/oddb/util/server.rb: def run_updater
def initialize(*args)
super
@rss_mutex = Mutex.new
run_exporter if(ODDB.config.run_exporter)
run_updater if(ODDB.config.run_updater)
end
Notes
Question
def run_at(hour, &block)
Thread.new {
loop {
now = Time.now
run_at = Time.local(now.year, now.month, now.day, hour)
while(now > run_at)
run_at += 24*60*60
end
sleep(run_at - now)
block.call
}
}
end
def run_updater
@updater = run_at(ODDB.config.update_hour) { Updater.run }
end
Notes
Experiment
lib/oddb/util/server.rb
def run_at(hour, &block)
Thread.new {
loop {
now = Time.now
print "now="
p now
run_at = Time.local(now.year, now.month, now.day, hour)
print "run_at="
p run_at
while(now > run_at)
run_at += 24*60*60
print "updated run_at="
p run_at
end
print "sleep length=", (run_at - now)/60/60, " [h]\n"
sleep(run_at - now)
block.call
}
}
end
Result
now=Fri Oct 22 15:53:31 +0200 2010 now=Fri Oct 22 15:53:31 +0200 2010 run_at=Fri Oct 22 04:00:00 +0200 2010 #=> run_exporter run_at=Fri Oct 22 02:00:00 +0200 2010 #=> run_updater updated run_at=Sat Oct 23 04:00:00 +0200 2010 #=> run_exporter updated run_at=Sat Oct 23 02:00:00 +0200 2010 #=> run_updater sleep length=12.107931385 [h] #=> run_exporter sleep length=10.1079313686111 [h] #=> run_updater
Notes
Hypothesis
bin/oddbd
begin
server = ODDB::Util::Server.new
server.extend(DRbUndumped)
@server = server
url = @config.server_url
url.untaint
DRb.start_service(url, server)
$SAFE = 1
logger.info('start') {
sprintf("starting oddb-server on %s", url) }
DRb.thread.join
rescue Exception => error
logger.error('fatal') { error }
raise
end
lib/oddb/util/server.rb
def run_at(hour, &block)
Thread.new {
loop {
now = Time.now
run_at = Time.local(now.year, now.month, now.day, hour)
while(now > run_at)
run_at += 24*60*60
end
sleep(run_at - now)
block.call
}
}
end
def run_updater
@updater = run_at(ODDB.config.update_hour) { Updater.run }
end
Hypothesis
Question
Refer