/* Copyright 2010 Aaron J. Radke Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cc.drx import java.io.{InputStream => JIS} import java.io.{OutputStream => JOS} import java.util.zip.GZIPOutputStream import java.util.zip.GZIPInputStream import java.nio.charset.Charset import java.nio.charset.StandardCharsets.UTF_8 2017-07-30("use Output or Input or StreamGenerator instead","dhu") object IO{ //--Output constructors/wrappers/generators 2017-07-30("use Output.apply instead","dhu") def apply(os:JOS):Output = Output(os) 2017-07-30("use Output.bytes instead","dhu") def bytes:Output.ByteStreamOutput = Output.bytes // //--Input constructors/wrappers/generators 2017-07-30("use Input.apply instead","dhu") def apply(bytes:Array[Byte]):Input = new Input(new java.io.ByteArrayInputStream(bytes)) 2017-07-30("use Input.apply instead","dhu") def apply(is:JIS):Input = new Input(is) 2017-07-30("use Input.apply instead","dhu") def apply(string:String):Input = Input(string, UTF_8) 2017-07-30("use Input.apply instead","dhu") def apply(string:String,charset:Charset):Input = Input(string getBytes charset) // def apply(url:URL):Input = apply(url.toJava.openStream) 2017-07-30("use Input.apply instead","dhu") def apply(url:java.net.URL):Input = Input(url.openStream) //TODO this maybe need to be removed in preference to URL 2017-07-30("use Input.url instead","dhu") def url(uri:String):Input = Input(new java.net.URL(uri)) 2017-07-30("use Input.resource instead","dhu") def resource(path:String):Input = Input.resource(File(path)) 2017-07-30("use Input.resource instead","dhu") def resource(file:File):Input = Input.resource(file) } //--Input and output generators //----------------------- trait IOGenerator[I,O]{ val exts:List[String] def canGenerate(file:File):Boolean = canWrite && exts.exists{ext => file.name.endsWith("."+ext)} def canRead(file:File):Boolean = canGenerate(file) def canWrite(file:File):Boolean = canGenerate(file) def canWrite:Boolean = true //default assume the stream can be written as well as read def apply(in:Input):I = apply(in.is) def apply(out:Output):O = apply(out.os) def apply(jis:JIS):I// = is(jis) def apply(jos:JOS):O// = os(jos) } trait StreamGenerator extends IOGenerator[Input,Output]{ def apply(bytes:Array[Byte]):Input = apply(new java.io.ByteArrayInputStream(bytes)) def apply(string:String,charset:Charset=UTF_8):Input = apply(string getBytes charset) override def apply(in:Input):Input = in //Input(in.is) override def apply(out:Output):Output = out //Output(out.os) } object StreamGenerator{ case object Normal extends StreamGenerator{ val exts:List[String] = List.empty def apply(jis:JIS) = Input(jis) def apply(jos:JOS) = Output(jos) } case object GZ extends StreamGenerator{ val exts:List[String] = List("gz") def apply(jis:JIS) = Input(new GZIPInputStream(jis)) def apply(jos:JOS) = Output(new GZIPOutputStream(jos)) } } //--Output stream constructors object Output{ def apply(os:JOS):Output = new Output(os) def bytes:Output.ByteStreamOutput = new ByteStreamOutput(new java.io.ByteArrayOutputStream) def nullOutput:Output = new Output( new JOS{ override def write(b:Int):Unit = () }) def toByteArray(write: Output => Unit):Array[Byte] = {val out = Output.bytes; write(out); out.toByteArray} class ByteStreamOutput(bos:java.io.ByteArrayOutputStream) extends Output(bos){ def toByteArray = bos.toByteArray } //-- bytebuffer def apply(buf:java.nio.ByteBuffer):Output = Output(new JOSByteBuffer(buf)) //TODO write test for this since it was not required right now but fit the patter of the input bytebuffer wrapper... class JOSByteBuffer(buf:java.nio.ByteBuffer) extends JOS{ // simple code example to wrap a bytebuffer as an outputstream https://stackoverflow.com/a/6603018/622016 def write(byte:Int):Unit = {buf.put(byte.toByte); ()} override def write(bytes:Array[Byte], offset:Int, len:Int):Unit = {buf.put(bytes, offset, len); ()} } } //--Output stream wrappers and methods class Output(val os:JOS){ //TODO wrap autoclose with proper try finally blocks that really close all the resources like Shabin suggestions with Scope at ScalaWorld2016 var autoClose = true def as(sg:StreamGenerator):Output = {val out = sg(os); out.autoClose = autoClose; out} //pass through autoClose option def keepOpen = {autoClose = false; this} //turn off autoClose def apply[A](applyFunc:JOS => A):Try[A] = { val res = Try{applyFunc(os)} if(autoClose) os.close //TODO need to clean up if there was a failure to close and use pat in others res } def write(bytes:Array[Byte]):Unit = { os.write(bytes) if(autoClose) os.close } def write(writeFunc:java.io.DataOutputStream => Unit):Unit = { val dos = new java.io.DataOutputStream(os) writeFunc(dos) if(autoClose) dos.close } def print(printFunc:java.io.PrintWriter => Unit):Unit = { val pw = new java.io.PrintWriter(os) printFunc(pw) if(autoClose) pw.close } def print(string:String):Unit = print(f => f.print(string)) def println(string:String):Unit = print(f => f.println(string)) def print(strings:Iterable[String]):Unit = print{f => for(s <- strings) f.println(s)} def close = os.close } object Input{ lazy val std = new Input(System.in) //--Input constructors/wrappers/generators def apply(is:JIS):Input = new Input(is) def apply(bytes:Array[Byte]):Input = new Input(new java.io.ByteArrayInputStream(bytes)) def apply(string:String):Input = Input(string, UTF_8) def apply(string:String,charset:Charset):Input = Input(string getBytes charset) // def apply(url:URL):Input = apply(url.toJava.openStream) def apply(url:java.net.URL):Input = Input(url.openStream) //TODO this maybe need to be removed in preference to URL def url(uri:String):Input = Input(new java.net.URL(uri)) // 2017-07-30("use resource(path).get instead to emphasize unsafe pluck","2017-12-05") // def resource(path:String):Input = resource(path).get // 2017-07-30("use resource(file).get instead to emphasize unsafe pluck","2017-12-05") // def apply(bytes:Iterator[Byte]):Input = new Input(/*new java.io.BufferedInputStream*/(new JIS{ // private var buf:Array[Byte] = bytes.take(1.k).toArray // private var i:Int = 0 def read():Int = if(bytes.hasNext) bytes.next().toInt else -1 //TODO add more JIS functionality for higher performance override def available:Int = if(bytes.hasNext) 1 else 0 //the default implementation=0 so could break some functions //TODO add more JIS functionality for higher performance //FIXME to work with AIS (audio) })) def apply(bytes:Iterable[Byte]):Input = apply(bytes.iterator) def apply(buf:java.nio.ByteBuffer):Input = Input(new JISByteBuffer(buf)) class JISByteBuffer(buf:java.nio.ByteBuffer) extends JIS{ // simple code example to wrap a bytebuffer as an inputstream https://stackoverflow.com/a/6603018/622016 //TODO @throws(classOf[IOException]) def read():Int = if(buf.hasRemaining) buf.get & 0xFF else -1 //TODO @throws(classOf[IOException]) override def read(bytes:Array[Byte], offset:Int, len:Int):Int = { if(buf.hasRemaining){ val shortestLength = len min buf.remaining buf.get(bytes, offset, shortestLength) shortestLength } else -1 } } private def resourceURL(file:File):Option[java.net.URL] = { val path = file.unixPath val modPath = if(path startsWith "/") path else "/"+path Option(getClass.getResource(modPath)) //wrap the null return as an None } def resource(file:File):Input = resourceOption(file).get def resourceOption(file:File):Option[Input] = resourceURL(file) map Input.apply def isResource(file:File):Boolean = resourceURL(file).isDefined // 2017-07-30("used resource(file.path).get instead", "2017-12-05") //deprecation warnings were not caught and the type checker complained instead // def resource(path:String):Input = Input(resourceURL(File(path)).get) def empty:Input = Input("") // def apply(file:File) = TODO add a smart searhc for local file then resources... //--implicits import scala.language.implicitConversions implicit def Input2InputStream(input:Input):JIS = input.is //--Misc data funtions // def bytes(xs:Int*):Array[Byte] = xs map(_.toByte).toArray // //def buffer(xs:Byte*):java.nio.ByteBuffer /* TODO remove in favor of xxd def xdd(bs:Array[Byte], cols:Int=16, bytesPerGroup:Int=2):Iterator[String] = { val bytesPerLine = bytesPerGroup*cols bs.grouped(bytesPerLine).zipWithIndex.map{case (l,i) => val index = "%08X" format i*bytesPerLine val hex = l grouped bytesPerGroup map {g => g map {"%02X" format _} mkString ""} mkString " " val utf = l grouped 2 map {g => U16(g(0), g(1)).toInt.toChar} mkString "" val padding = " "*((bytesPerGroup*2+1)*cols - hex.size) index + " " + hex + padding + " " + Ansi.strip(utf).replaceAll("[\r\n]",".") } } */ class LineIterator(is:JIS, debug:Option[Int]=None, autoClose:Boolean=true) extends Iterator[String]{ private val src = Input(is).bufferedSource(scala.io.Source.DefaultBufSize*8) private val it0 = src.getLines() private val it = if(debug.isDefined) it0 take debug.get else it0 private var isOpen = true def hasNext:Boolean = if(isOpen && it.hasNext) true else {if(isOpen && autoClose) close; false} def next():String = it.next() def close = { isOpen = false; src.close} } } //---Input stream wrappers and methods class Input(val is:JIS){ //TODO wrap autoclose with proper try finally blocks that really close all the resources like Shabin suggestions with Scope at ScalaWorld2016 private var autoClose:Boolean = true def as(sg:StreamGenerator):Input = {val in = sg(is); in.autoClose = autoClose; in} //pass through autoClose option def keepOpen:Input = {val in = Input(is); in.autoClose = false; in} //turn off autoClose def apply[A](applyFunc:JIS => A):Try[A] = { val res = Try{applyFunc(is)} if(autoClose) is.close //TODO need to clean up if there was a failure to close and use pat in others res } def monitor(message:String="Progress"):Input = Input(new javax.swing.ProgressMonitorInputStream(null, message, is)) def close = is.close def consume:Unit = { val it = byteIt while(it.hasNext){it.next()} close } def read[A](readFunc:java.io.DataInputStream => A):A = { val dis = new java.io.DataInputStream(is) val res = readFunc(dis) if(autoClose) dis.close res } //FIXME remove this broken somewhat broken concept 2017-07-30("remove this broken somewhat broken concept, since the buffer size may not be full","dj") def readBytes(readFunc:Array[Byte] => Unit):Unit = { var bytesRead = 0 val buffSize = 8192*2 val buffer = new Array[Byte](buffSize) //TODO add isAvailable to pause? while({bytesRead = is.read(buffer); bytesRead != -1}) readFunc( if(bytesRead < buffSize) buffer take bytesRead else buffer ) if(autoClose) is.close } 2017-07-30("use byteIt, since bytes seems like an immutable list","dk") def bytes:Iterator[Byte] = byteIt(8192*2) /**default large buffer iterator chunks*/ def byteIt:Iterator[Byte] = byteIt(8192*2) def byteIt(bufferSize:Int):Iterator[Byte] = if(bufferSize <= 1) byteIt(1) else new Iterator[Byte]{ //--constants private val N = 8192*2 //buffer size //--mutables private val bs = new Array[Byte](bufferSize) //byte buffer private var n = 0 //bytes read private var i = bufferSize //buffer index private var foundEnd:Boolean = false //end of file cache private def eof:Boolean = { if(!foundEnd && n == -1){ if(autoClose) is.close //autoClose if the first time we found an eof foundEnd = true } foundEnd } private def eob:Boolean = i >= n-1 //end of buffer def hasNext:Boolean = { if(eof) false else if(eob) { n = is read bs //read the buffer i = -1 //reset buffer index // n != -1 !eof //check if eof } else true //inside buffer } def next():Byte = { // Log(bufferSize,n,i,foundEnd,eof,eob) if(foundEnd) ??? //no such element else { i += 1 bs(i) } } } /**readline like but with a callback on each charcter input; returning the full string*/ def readCharsWhile(callback:Char => Boolean,charset:Charset=UTF_8):Unit = { val reader = new java.io.BufferedReader(new java.io.InputStreamReader(is,charset)) var _char:Int = 0 var _continue = true while(_continue && {_char = is.read; /*print("[" + _char + "]");*/ _char != -1}) { //TODO add an available to prevent blocking on read _continue = callback(_char.toChar) } } /**readline like but with a callback on each charcter input; returning the full string*/ def promptLine:String= { var _string:String = "" readCharsWhile{ c => print(c) //show to std out if(c == '\n' || c == '\r') false else {_string += c ; true} } print("\n") _string } //TODO plan to deprecate since the copy direction is not explicilty clear 2017-07-30("use copyTo to be more specific where it is copied to", "dq") def copy(that:Output):Unit = copyTo(that) def copyTo(that:File):Unit = copyTo(that.out) def copyTo(that:Output):Unit = { //IO.copy(this.is, that.os, autoClose) var bytesRead = 0 val buffSize = 8192*2 val buffer = new Array[Byte](buffSize) while({bytesRead = this.is.read(buffer); bytesRead != -1}) that.os.write(buffer,0,bytesRead) that.os.flush if(this.autoClose) this.is.close if(that.autoClose) that.os.close } def lines:Input.LineIterator = new Input.LineIterator(is, None, autoClose) def lines(debug:Option[Int]):Input.LineIterator = new Input.LineIterator(is, debug, autoClose) def bufferedSource(bufferSize:Int=scala.io.Source.DefaultBufSize)(implicit codec:scala.io.Codec):scala.io.BufferedSource = scala.io.Source.createBufferedSource( is, bufferSize, reset = () => bufferedSource(bufferSize)(codec), close = () => is.close() ) (codec) private def toByteArrayOutputStream = { val bos = new java.io.ByteArrayOutputStream this copyTo Output(bos) bos } //don't override toString since it will consume the input on simple toString calls // 2017-07-30("dont override toString use `asString` instead","dq") // override def toString = "Input(<wrapper around an InputStream>)"// ??? //throws runtimeerror TODO throw compiletime eror at runtime... use [[asString]] instead def asString(charset:Charset) = new String(toByteArray,charset) def asString = new String(toByteArray, UTF_8) def toByteArray:Array[Byte] = toByteArrayOutputStream.toByteArray def toByteBuffer:java.nio.ByteBuffer = java.nio.ByteBuffer.wrap(toByteArray) def base64:String = Base64.encode(toByteArray) def cat:String = { if(is != null){ val buf = new java.io.BufferedReader(new java.io.InputStreamReader(is)) @tailrec def read(str:StringBuilder):StringBuilder= { val x = buf.readLine if(x != null && x.nonEmpty) read(str ++= x) else str } read(new StringBuilder).toString } else "" } import java.security.{MessageDigest,DigestInputStream} def digest(implicit md:MessageDigest):String = { //TODO should return a byteArray and byteArray has a toHex val digestBytes = try{ val dis = new DigestInputStream(is,md) val buffer = new Array[Byte](8192) while(dis.read(buffer) >= 0){} dis.close md.digest } finally {is.close} digestBytes.map("%02x" format _).mkString } def sha1 = digest(MessageDigest getInstance "SHA-1") def sha256 = digest(MessageDigest getInstance "SHA-256") def md5 = digest(MessageDigest getInstance "MD5") //-- support functions for hex dump xxd private def asHex(bs:Seq[Byte]):String = bs.map{"%02X" format _}.mkString("") private def asText(bs:Seq[Byte]):String = bs.map{b => if (b > 31) b.toChar.toString // printable most common case first ... else if (b < 0) "×" //extended else if (b == 0) "Ø" //null else if (b == 9) "\\t" //tab else if (b == 10) "\\n" //LF else if (b == 13) "\\r" //CR else "·" //control character //special glyphs supported by pdf standard Courier: ×Ø·¿¬®©¶ }.mkString("") /**Unix like hex dump as an efficient string iterator */ def xxd:Iterator[String] = { byteIt.grouped(16).zipWithIndex.map{case (bs,iLine) => val i = iLine*16 val h = bs.grouped(8).map{bs => bs.grouped(2).map(asHex).mkString(" ") }.mkString(" | ") val t = asText(bs) f"$i%08X | $h | $t" } } //-- support classes and functions for string seeking /**Unix like string iterator to finding ascii character sequences*/ class AsciiIterator(bs:Iterator[Byte]) extends Iterator[String]{ private def isCommonAscii(b:Byte):Boolean = b > 31 || b == 10 || b == 13 || b == 9 def hasNext:Boolean = bs.hasNext @tailrec final def next():String = { bs.dropWhile{b => !isCommonAscii(b)} val str = bs.takeWhile{b => isCommonAscii(b)}.map{_.toChar}.mkString("") if(str.size > 3) str else if(bs.hasNext) next() else "" } } /**Unix like string iterator to finding ascii character sequences*/ def strings:Iterator[String] = new AsciiIterator(byteIt) } //-- PathEntry is either a FileEntry or a DirEntry //TODO put all of these sub class inside some super trait like PathEntry or Input or File.Entry // TODO implment moving over zip without commons compress by using these PathEntry // so moving over zip files can work without the commons-compress dependency by using the built in java deflat/unzip sealed trait PathEntry{ //TODO this basic trait could be a superset of file val file:File val date:Date val size:Bytes } object PathEntry{ def apply(file:File):PathEntry = if(file.isDir) DirEntry(file, file.modified) else FileEntry(file) protected[drx] def normalizePath(path:String) = path.replace("\\","/") } object FileEntry{ def apply(file:File):FileEntry = new FileEntry(file, ()=>file.in, file.size, file.modified) def apply(file:File, content:String):FileEntry = new FileEntry(file, ()=>Input(content), Bytes(content.size), Date.now) def apply(file:File, bytes:Array[Byte]):FileEntry = new FileEntry(file, ()=>Input(bytes), Bytes(bytes.size), Date.now) } class FileEntry(val file:File, val makeStream:()=>Input, val size:Bytes, val date:Date) extends PathEntry{ override def toString = s"FileEntry($file, <Input>, size=$size, $date)" lazy val in:Input = makeStream() def as(sg:StreamGenerator):FileEntry = { val bos = Output.bytes in copyTo bos.as(sg) val bytes = bos.toByteArray new FileEntry(file, ()=>Input(bytes), Bytes(bytes.size), date) } def fullPath:String = PathEntry.normalizePath(file.path) } case class DirEntry(file:File, date:Date) extends PathEntry{ val size = Bytes(0L) //size:Long=ArchiveEntry.SIZE_UNKNOWN = -1 } class DeepFileEntry(val parent:FileEntry, e:FileEntry) extends FileEntry(e.file,e.makeStream,e.size,e.date){ /** parents are the parents in a walk deep*/ def parents:List[FileEntry] = parent match{ case p:DeepFileEntry => p :: p.parents case p:FileEntry => p :: Nil } /** file path as as a string (in a deep walk) */ override def fullPath:String = PathEntry.normalizePath((this :: parents).reverse map {_.file.path} mkString "/") def treePath:String = (this :: parents).reverse.zipWithIndex map {case (e,i) => " "*i + e.file.path} mkString "\n" override def toString = s"DeepFileEntry($fullPath, <Input>, size=$size, $date)" }