├── README.md ├── manual.rus.doc ├── mssql.yml.default ├── load32.rb ├── .gitignore ├── stat.rb ├── mssql.rb └── main.rb /README.md: -------------------------------------------------------------------------------- 1 | retro2af41 2 | ========== -------------------------------------------------------------------------------- /manual.rus.doc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ujifgc/retro2af41/master/manual.rus.doc -------------------------------------------------------------------------------- /mssql.yml.default: -------------------------------------------------------------------------------- 1 | --- 2 | development: 3 | adapter: sqlserver 4 | host: localhost 5 | database: db 6 | username: user 7 | password: pass 8 | -------------------------------------------------------------------------------- /load32.rb: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | require 'win32ole' 3 | require 'awesome_print' 4 | require 'yaml' 5 | require 'tiny_tds' 6 | require 'progressbar' 7 | require 'logger' 8 | 9 | load 'mssql.rb' 10 | load 'main.rb' 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | *.rbc 3 | .bundle 4 | .config 5 | coverage 6 | InstalledFiles 7 | lib/bundler/man 8 | pkg 9 | rdoc 10 | spec/reports 11 | test/tmp 12 | test/version_tmp 13 | tmp 14 | log*/* 15 | 16 | # YARD artifacts 17 | .yardoc 18 | _yardoc 19 | doc/ 20 | 21 | # local config 22 | mssql.yml 23 | -------------------------------------------------------------------------------- /stat.rb: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | 3 | j = 0 4 | Dir.open('.').each do |dir| 5 | next unless dir.match(/^\d+\.done$/) 6 | p dir 7 | File.open "#{dir.to_i}.stat.log", 'w' do |out| 8 | out.write( "%5s\t%5s\t%8s\n" % ["Фонд", "Опись", "Записи"] ) 9 | j = 0 10 | Dir.open(dir).sort_by{|n|n.split('.')[0].split('-')[0].to_i rescue 0}.each do |name| 11 | name.encode! 'utf-8' 12 | if name.match /\.txt$/ 13 | buf = File.read( dir + '/' + name ) 14 | k = 0 15 | buf.each_line do |line| 16 | if line.match /\d+\t.*$/ 17 | k += 1 18 | else 19 | p line 20 | end 21 | end 22 | fund, inv = name.rpartition('.')[0].split('-') 23 | if inv[-1].match /\d/ 24 | out.write( "%5s\t%5s\t%8d\n" % [fund.to_s, inv.to_s, k] ) 25 | else 26 | out.write( "%5s\t%6s\t%8d\n" % [fund.to_s, inv.to_s, k] ) 27 | end 28 | j += k 29 | end 30 | end 31 | out.write "Итого\tзаписей\t%8d\n" % [j] 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /mssql.rb: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | class MSSQL 3 | attr_accessor :connection 4 | 5 | def initialize( opts = {} ) 6 | yml = YAML.load_file('mssql.yml')['development'] 7 | @parameters = {} 8 | yml.each{ |k,v| @parameters[k.to_sym] = v.to_s } 9 | @connection = nil 10 | @log = Logger.new $dname + '/mssql-query.log' 11 | @logi = Logger.new $dname + '/mssql-insert.log' 12 | end 13 | 14 | def connect 15 | @connection = TinyTds::Client.new @parameters 16 | if @connection.closed? 17 | raise Exception, 'connection error' 18 | else 19 | ap "Соединение успешно" 20 | end 21 | end 22 | 23 | def close 24 | @connection.close 25 | end 26 | 27 | def execute( q ) 28 | @log.info 'EXECUTE ' + q.gsub(/[\n\r\s]+/, ' ') 29 | @connection.execute q 30 | end 31 | 32 | def delete( q ) 33 | @log.info 'DELETE ' + q.gsub(/[\n\r\s]+/, ' ') 34 | result = @connection.execute q 35 | result.do 36 | end 37 | 38 | def insert( q ) 39 | @logi.info 'INSERT ' + q.gsub(/[\n\r\s]+/, ' ') 40 | result = @connection.execute q 41 | result.insert 42 | end 43 | 44 | def part 45 | @logi.info ' ' 46 | @logi.info ' ' 47 | end 48 | 49 | def query_table( q ) 50 | @log.info 'TABLE ' + q.gsub(/[\n\r\s]+/, ' ') 51 | connect if !@connection || @connection.closed? 52 | result = @connection.execute q 53 | table = result.to_a 54 | result.cancel 55 | table 56 | end 57 | 58 | def query_value( q ) 59 | @log.info 'VALUE ' + q.gsub(/[\n\r\s]+/, ' ') 60 | connect if !@connection || @connection.closed? 61 | result = @connection.execute q 62 | value = result.first.first[1] 63 | result.cancel 64 | value.kind_of?( Array ) ? value[1] : value 65 | end 66 | alias value query_value 67 | 68 | def query( q ) 69 | @log.info 'BLOCK ' + q.gsub(/[\n\r\s]+/, ' ') 70 | if block_given? 71 | connect if !@connection || @connection.closed? 72 | result = @connection.execute q 73 | result.each do |row| 74 | yield row 75 | end 76 | result.cancel 77 | else 78 | raise Exception, 'no blocks given' 79 | end 80 | end 81 | 82 | end 83 | -------------------------------------------------------------------------------- /main.rb: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | 3 | # ATTENTION! If you are using non-ascii characters in your database, 4 | # please invoke `chcp 65001 before using this script 5 | 6 | $dname = "log-#{DateTime.now.strftime("%Y%m%d%H%M%S")}" 7 | Dir.mkdir $dname 8 | 9 | trap :INT do 10 | puts "\nBreak requested. Wait for it" 11 | @stop = true 12 | end 13 | 14 | ms = MSSQL.new 15 | ms.connect 16 | 17 | login_query = " 18 | create table #t_user_var( 19 | uisn integer, 20 | lisn integer null, 21 | prot_flags varchar(255) default ',' ) 22 | 23 | INSERT INTO #t_user_var (uisn) 24 | SELECT isn_lclassif 25 | FROM USER_CL 26 | WHERE upper(ORACLE_ID)='AF41' 27 | " 28 | ms.execute login_query 29 | 30 | absent = {} 31 | absent[:fund] = {} 32 | absent[:inventory] = {} 33 | 34 | present = {} 35 | present[:unit] = {} 36 | error = {} 37 | 38 | new_unit_isn = 1 + ms.value( 'SELECT MAX(ISN_UNIT) FROM UNIT' ).to_i 39 | new_weight = 1 + ms.value( 'SELECT MAX(WEIGHT) FROM UNIT' ).to_i 40 | 41 | new_unit_isn = 400001 if new_unit_isn <= 400000 42 | new_weight = 200001 if new_weight <= 200000 43 | 44 | @logF = Logger.new $dname + '/search-fund.log' 45 | @logI = Logger.new $dname + '/search-inventory.log' 46 | @logU = Logger.new $dname + '/search-unit.log' 47 | @logB = Logger.new $dname + '/binary.log' 48 | 49 | txt_dir = ARGV[0] 50 | unless txt_dir.to_s.length > 0 51 | ap "Не указан каталог с описями" 52 | exit 53 | end 54 | files = Dir.glob("#{txt_dir}/*.txt") 55 | unless files.count > 0 56 | ap "В каталоге '#{txt_dir}' нет файлов описей '*.txt'" 57 | exit 58 | end 59 | 60 | gbar = ProgressBar.new('all', 1) 61 | files.each do |file| 62 | break if @stop 63 | file.encode! 'UTF-8' 64 | begin 65 | query = '' 66 | line = '' 67 | 68 | fund_name, inventory_name = File.basename(file, '.*').split('-') 69 | lines = File.read(file).force_encoding('UTF-8').lines.to_a 70 | pbar = ProgressBar.new(file, lines.count - 3) 71 | 72 | fund_result = ms.query_table "SELECT * FROM FUND WHERE FUND_NUM_2='#{fund_name}'" 73 | if fund_result.any? 74 | fund = fund_result.first 75 | else 76 | @logF.error "Внимание: фонд #{fund_name} не найден" 77 | absent[:fund].merge! fund_name => fund_name 78 | end 79 | 80 | inventory_name_1 = inventory_name.to_i 81 | inventory_name_2 = inventory_name.gsub(/\d+/,'') 82 | q = "SELECT * FROM INVENTORY WHERE ISN_FUND='#{fund['ISN_FUND']}' AND INVENTORY_NUM_1='#{inventory_name_1}' AND ISNULL(INVENTORY_NUM_2, '')='#{inventory_name_2}'" 83 | inventory_result = ms.query_table q 84 | if inventory_result.any? 85 | inventory = inventory_result.first 86 | inventory_isn = inventory['ISN_INVENTORY'] 87 | else 88 | @logI.error "Внимание: опись #{inventory_name} фонда #{fund_name} не найдена" 89 | absent[:inventory].merge! inventory_name => [inventory_name_1, inventory_name_2] 90 | end 91 | 92 | unit_result = ms.query_table "SELECT * FROM UNIT WHERE ISN_INVENTORY='#{inventory['ISN_INVENTORY']}'" 93 | if unit_result.any? 94 | @logU.warn "Внимание: фонд #{fund_name.to_s.rjust(6)}, опись #{inventory_name.to_s.rjust(3)} уже содержит #{unit_result.count.to_s.rjust(4)} дел, файл #{file.to_s.ljust(12)} пропущен" 95 | present[:unit].merge! file => unit_result 96 | next 97 | else 98 | @logU.info "Внесение #{lines[4..-1].count} строк из файла #{file}" 99 | end 100 | 101 | lines[4..-1].each do |line| 102 | 103 | nn, name, dates, lists, info = line.strip.split(/\t/) 104 | dates = dates.split('-') rescue [] 105 | 106 | unit_name = name.strip.gsub "'", "''" 107 | unit_num_1 = nn.to_i 108 | unit_num_2 = nn.gsub(/\d+/,'') 109 | year_a = "'"+dates[0].rpartition('.').last+"'" rescue 'NULL' 110 | year_b = "'"+dates[1].rpartition('.').last+"'" rescue 'NULL' 111 | unit_page_count = lists.to_i 112 | unit_dates = dates.join(' - ') 113 | 114 | # here goes SQL MAGIC 115 | 116 | query = " 117 | INSERT INTO [dbo].[UNIT] 118 | ([ISN_UNIT] ,[ISN_HIGH_UNIT] ,[ISN_INVENTORY] ,[ISN_DOC_TYPE] ,[ISN_LOCATION] 119 | ,[ISN_SECURLEVEL] ,[SECURITY_CHAR] ,[SECURITY_REASON] ,[ISN_INVENTORY_CLS] ,[ISN_STORAGE_MEDIUM] 120 | ,[ISN_DOC_KIND] ,[UNIT_KIND] ,[UNIT_NUM_1] ,[UNIT_NUM_2] ,[VOL_NUM] 121 | ,[NAME] ,[ANNOTATE] ,[DELO_INDEX] ,[PRODUCTION_NUM] ,[UNIT_CATEGORY] 122 | ,[NOTE] ,[IS_IN_SEARCH] ,[IS_LOST] ,[HAS_SF] ,[HAS_FP] 123 | ,[HAS_DEFECTS] ,[ARCHIVE_CODE] ,[CATALOGUED] ,[WEIGHT] ,[UNIT_CNT] 124 | ,[START_YEAR] ,[START_YEAR_INEXACT] ,[END_YEAR] ,[END_YEAR_INEXACT] ,[MEDIUM_TYPE] 125 | ,[BACKUP_COPY_CNT] ,[HAS_TREASURES] ,[IS_MUSEUM_ITEM] ,[PAGE_COUNT] ,[CARDBOARDED] 126 | ,[ADDITIONAL_CLS] ,[ALL_DATE] ,[ISN_SECURITY_REASON]) 127 | VALUES ( 128 | '#{new_unit_isn}',NULL,'#{inventory_isn}',1,NULL, 129 | 1,'o',NULL,'#{inventory_isn}',NULL, 130 | NULL,703,'#{unit_num_1}','#{unit_num_2}',NULL, 131 | '#{unit_name}',NULL,NULL,NULL,'b', 132 | NULL,'N','N','N','N', 133 | 'N',NULL,'N','#{new_weight}',NULL, 134 | #{year_a},NULL,#{year_b},NULL,'T', 135 | NULL,'N','N','#{unit_page_count}',NULL, 136 | '','#{unit_dates}',NULL) 137 | " 138 | 139 | new_unit_isn += 1 140 | new_weight += 1 141 | 142 | ms.insert query 143 | 144 | pbar.inc 145 | 146 | # end SQL MAGIC 147 | 148 | end 149 | pbar.finish 150 | ms.part 151 | rescue TinyTds::Error => e 152 | ap e 153 | error[new_unit_isn] = [e,query,line] 154 | @logU.error [e,query,line].inspect 155 | end 156 | end 157 | gbar.finish 158 | 159 | @logB << "\r\n-- absent --\r\n" + absent.inspect 160 | @logB << "\r\n-- present --\r\n" + present.inspect 161 | @logB << "\r\n-- error --\r\n" + error.inspect 162 | 163 | ms.close 164 | --------------------------------------------------------------------------------