class ExtremeZip
Public Class Methods
new()
click to toggle source
# File lib/extremezip.zzaqsv.rb, line 19 def initialize @wholeCbor = {} # 整个CBOR结构 @vfsDataList = [] # 数据块压缩块列表 @filePartCounter = 0 # 文件分块计数器 @responsePipeList = [] # 任务回复管道列表 @processIdList = [] # 子进程编号列表。 @maxSubProcessAmount = Etc.nprocessors # 获取最大的子进程个数 @dataBlockLength = 33554432 # 数据块单元长度, 32MiB @clipDownloader = VictoriaFresh.new # 创建下载器。 @clipDownloader.diskFlush = true # 向磁盘写入缓存 @clipDownloader.diskMultiFile = true # 写多个磁盘文件 @clipDownloader.diskFileName = 'victoriafreshdata.v.' # 磁盘文件名前缀 @clipDownloader.diskFlushSize = @dataBlockLength # 磁盘文件大小 end
Public Instance Methods
addBasicFileInformation()
click to toggle source
加入基本文件信息
# File lib/extremezip.zzaqsv.rb, line 49 def addBasicFileInformation @wholeCbor['version'] = 68 # 文件格式版本号 uuid = UUID.new # 获取生成器 @wholeCbor['uuid'] = uuid.generate # 指定本个压缩包的唯一编号 end
compressInSubProcess(currentBlockData, currentResponsePipe)
click to toggle source
在子进程中具体执行的压缩代码
# File lib/extremezip.zzaqsv.rb, line 159 def compressInSubProcess(currentBlockData, currentResponsePipe) checkMemoryUsage(115) # Debug currentBlockDataToCompress = currentBlockData # 读取数据块 currentCompressedVfsData = LZMA.compress(currentBlockDataToCompress) # 压缩当前块 checkMemoryUsage(120) puts("compressed data length: #{currentCompressedVfsData.bytesize}") # Debug. currentResponsePipe.put currentCompressedVfsData # 将压缩后的数据块写入到回复管道中 checkMemoryUsage(125) puts("finished #{Process.pid}") # Debug end
compressVfsMenu(victoriaFresh)
click to toggle source
压缩目录数据。
# File lib/extremezip.zzaqsv.rb, line 39 def compressVfsMenu(victoriaFresh) replyByteArray = victoriaFresh.to_cbor # #打包成字节数组。 # 压缩目录数据并放入CBOR: compressedVfsMenu = LZMA.compress(replyByteArray) # 压缩目录数据 @wholeCbor['vfsMenu'] = compressedVfsMenu # 加入目录 end
exz(rootPath)
click to toggle source
压缩
# File lib/extremezip.zzaqsv.rb, line 57 def exz(rootPath) victoriaFresh, = @clipDownloader.checkOnce(rootPath) # 打包该目录树。 @filePartAmount = @clipDownloader.currentDiskFlushSuffix # 获取文件个数 compressVfsMenu(victoriaFresh) # 压缩目录数据。 addBasicFileInformation # 加入基本文件信息 processIdList, responsePipeList = launchSubProcesses # 启动子进程。 receiveCompressedVfsDataList(processIdList, responsePipeList) # 接收压缩后的数据块列表 checkMemoryUsage(155) @wholeCbor['vfsDataList'] = @vfsDataList # 加入数据 wholeFileContent = 'exz' + "\0" + @wholeCbor.to_cbor # 追加CBOR字节数组 # 写入文件: writeFile(wholeFileContent, victoriaFresh) # 写入文件内容 end
launchSubProcesses()
click to toggle source
启动子进程。
# File lib/extremezip.zzaqsv.rb, line 147 def launchSubProcesses #while ((@filePartCounter < @filePartAmount) && (true)) # 未处理完毕,并且未达到最大子进程个数 while ((@filePartCounter < @filePartAmount) && (@filePartCounter<@maxSubProcessAmount)) # 未处理完毕,并且未达到最大子进程个数 currentResponsePipe, p1 = schedule1Block(@filePartCounter) # 计划一个块的压缩计算 end # while processDataLength < victoriaFreshData.byte_size do #未处理完毕 [@processIdList, @responsePipeList] end
readBlockFile(filePartCounter)
click to toggle source
读取块文件内容
# File lib/extremezip.zzaqsv.rb, line 107 def readBlockFile(filePartCounter) currentBlockFile = File.new(@clipDownloader.diskFileName + filePartCounter.to_s, 'rb') # 打开文件 currentBlockData = currentBlockFile.read # 读取全部内容 currentBlockFile.close # 关闭文件 File.delete(currentBlockFile) # 删除数据块文件 currentBlockData end
receiveCompressedVfsDataList(processIdList, responsePipeList)
click to toggle source
接收压缩后的数据块列表
# File lib/extremezip.zzaqsv.rb, line 87 def receiveCompressedVfsDataList(processIdList, responsePipeList) processCounter = 0 # 子进程计数器 while (processCounter<@filePartAmount) # 并不是所有分块都被处理完毕了。 currentSubProcess=processIdList[processCounter] # 获取子进程对象 #processIdList.each do |currentSubProcess| compressed = receiveFromSubProcess(currentSubProcess, responsePipeList, processCounter) # 从子进程中读取数据,并终止子进程 @vfsDataList << compressed # 加入数据块列表中 checkMemoryUsage(150) processCounter += 1 # 子进程计数 if (@filePartCounter<@filePartAmount) # 还有一些分块尚未交给子进程进行处理 schedule1Block(@filePartCounter) # 再启动一个子进程 end # if (@filePartCounter<@filePartAmount) # 还有一些分块尚未交给子进程进行处理 end # processIdList.each do |currentSubProcess| end
receiveFromSubProcess(currentSubProcess, responsePipeList, processCounter)
click to toggle source
从子进程中读取数据,并终止子进程
# File lib/extremezip.zzaqsv.rb, line 175 def receiveFromSubProcess(currentSubProcess, responsePipeList, processCounter) puts("waiting #{currentSubProcess}") # Debug checkMemoryUsage(140) currentResponsePipe = responsePipeList[processCounter] # 任务回复管道 currentCompressedVfsDataFromSubProcess = currentResponsePipe.get # 读取压缩后数据 checkMemoryUsage(145) Process.waitpid(currentSubProcess) # 等待该个子进程 currentCompressedVfsDataFromSubProcess end
schedule1Block(filePartCounter)
click to toggle source
计划一个块的压缩计算
# File lib/extremezip.zzaqsv.rb, line 120 def schedule1Block(filePartCounter) currentBlockData = readBlockFile(filePartCounter) # 读取块文件内容 # currentTaskPipe = Cod.pipe # 任务分配管道 currentResponsePipe = Cod.pipe # 任务回复管道 puts("forking sub process, file part counter: #{filePartCounter}") # Debug. p1 = fork do # 复制出子进程 compressInSubProcess(currentBlockData, currentResponsePipe) # 在子进程中具体执行的压缩代码 end # p1 = fork do #复制出子进程 # processDataLength += @dataBlockLength # 计数 checkMemoryUsage(130) # 记录管道: # taskPipeList << currentTaskPipe @responsePipeList << currentResponsePipe # 记录回复管道 @processIdList << p1 # 记录到子进程列表中 @filePartCounter += 1 # 计数 [currentResponsePipe, p1] end
writeFile(wholeFileContent, victoriaFresh)
click to toggle source
写入文件内容
# File lib/extremezip.zzaqsv.rb, line 80 def writeFile(wholeFileContent, victoriaFresh) extremeZipOutputFile = File.new("#{victoriaFresh['name']}.exz", 'wb') # 创建文件 extremeZipOutputFile.syswrite(wholeFileContent) # 写入文件 extremeZipOutputFile.close # 关闭文件 end