/* Copyright 2010 Aaron J. Radke Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cc.drx import scala.collection.JavaConverters._ import scala.concurrent.blocking object Token{ def find(key:String):Option[Token] = { val fullKey = key.toUpperCase +"_TOKEN" OS.Env.get(fullKey) map Token.apply } def github = find("github") } case class Token(value:String) object Mime{ final class MimeStringContext(val sc:StringContext) extends AnyVal{ def mime(args:Any*):Mime = Mime(sc.s(args:_*)) } private lazy val fromExt:Map[String,Mime] = { for( (mime,exts) <- fromMime; ext <- exts) yield ext -> mime }.toMap private lazy val fromMime:Map[Mime,List[String]] = {for ( in <- Input.resourceOption(file"mime.types").toList; //Option is safe if mime.types is not available line <- in.lines; //assumes mime.types exists args = line.trim split "\\s+"; if args.size > 1; mime = new Mime(args.head); //the only way to make new Mime is using the new operator ext <- args //include the map to itself if it already exists ) yield mime -> args.toList}.toMap val default:Mime = new Mime("application/octet-stream") def extsOf(mime:Mime):List[String] = fromMime.get(mime) getOrElse Nil def get(ext:String):Option[Mime] = fromExt.get(ext) def apply(ext:String):Mime = get(ext) getOrElse default //least specific is default if not found } class Mime(val name:String){ override def toString = s"Mime($name)" def exts:List[String] = Mime.extsOf(this) def typ:String = name.split('/').head //TODO make this more robust def subtyp:String = name.split('/').last //TODO make this more robust } object URL{ implicit object ParsableURL extends Parsable[URL]{ def apply(v:String):URL = URL(v) } def apply(url:String):URL = apply(url, Some("https")) def apply(url:String,defaultProtocol:Option[String]):URL = { val autoURL = defaultProtocol map {proto => if(url contains "://") url else proto+"://"+url} getOrElse url new URL(new java.net.URL(autoURL)) } val Href = Pluck("""href\s*="(.+?)"""".r) final class URLStringContext(val sc:StringContext) extends AnyVal{ def url(args:Any*):URL = URL(sc.s(args:_*)) def http(args:Any*):URL = URL(sc.s(args:_*)).copy(protocol="http") def https(args:Any*):URL = URL(sc.s(args:_*)).copy(protocol="https") } //old java style class construct to make it work is ugly TODO wrap this as a function which returns a Future of the type trait Connectable[A]{ def apply(c:Connection):A } object Connectable{ implicit case object ConnectableString extends Connectable[String]{ def apply(c:Connection):String = c.in.asString } implicit case object ConnectableJson extends Connectable[Json]{ def apply(c:Connection):Json = Json(c.in) } implicit case object ConnectableFile extends Connectable[File]{ def apply(c:Connection):File = { val f = File(c.url.file.map{_.name} getOrElse "un-named") c.in copyTo f.out f } } } // Such an excelent guiding SO examples at https://stackoverflow.com/a/2793153/622016 class Connection(val url:URL, val timeout:Time=20.s) extends StringMap{ private val c = url.url.openConnection // get the java.net.URLConnection //--GET set request properties BEFORE connect if(url.postData.isEmpty) url.props.foreach{case (k,v) => c.setRequestProperty(k,v) } c.setConnectTimeout(timeout.ms.toInt) c.setReadTimeout(timeout.ms.toInt) // c.setChunkedStreamingMode(7000) //on http c.setDoInput(true) //true is default //--set request properties AFTER connect https://stackoverflow.com/a/4206094/622016 for(postInput <- url.postData){ // c setRequestMethod "POST" // c setInstanceFollowRedirects false c setDoOutput true c setUseCaches false val postProps:Map[String,String] = if(url.props contains "Content-Type") url.props else url.props + ("Content-Type" -> "application/x-www-form-urlencoded; charset=utf-8") //TODO set content length if from a known file length // c.setRequestProperty( "Content-Length", Integer.toString( postDataLength )) // c.setRequestProperty( "Accept-Charset", "UTF-8") //TODO cleverly set acceptable charset postProps.foreach{case (k,v) => c.setRequestProperty(k,v) } val out = new Output(c.getOutputStream) postInput copyTo out } //c.connect() //if not already connected (Note: this breaks a POST if it is called before the post data // //writing to c.getOutputStream and reading from c.getInputStream does the actual connect call // val in:Input = new Input(c.getInputStream) //this is val always kick off reading (and not allow new ins) lazy val content = in.asString //TODO think if this should be kept around//if used it breaks the inputStream def responseCode:Option[Int] = for(resp <- header.get("<null>"); v <- resp.headOption; i <- Try{(v.trim split " " apply 1).toInt}.toOption) yield i private def safe(k:String) = Option(k).getOrElse("<null>") private lazy val header:Map[String,List[String]] = Java.toScala(c.getHeaderFields).map{ case (k,vs) => safe(k) -> Java.toScala(vs).toList }.toMap def getString(key:String):Option[String] = normHeader get key.toLowerCase lazy val keys = normHeader.keys.toList private lazy val normHeader = header.map{case (k,v) => k.toLowerCase -> v.mkString(", ")} def niceHeader:String = header.map{case (k,v) => (k+":").fit(30) + v.mkString(", ")}.toList.sorted mkString "\n" //Date TODO add date parser type from the http url date format //https://tools.ietf.org/html/rfc2616#page-20 //http://stackoverflow.com/a/8642463/622016 //DateTimeFormatter.RFC_1123_DATE_TIME. def modified:Date = Date(c.getLastModified) def date:Date = Date(c.getDate) def expiration:Date = Date(c.getExpiration) def encoding:String = c.getContentEncoding //--common scrape/plucks private val RelLink = Pluck("""<([^>]+)>\s*;\s*rel\s*="([^"]+)"""".r, 1, 2) lazy val relLinks:Map[String,URL] = { for(link <- getString("link").toList; (a,b) <- link pluckAll RelLink) yield b -> URL(a) }.toMap def copyTo(file:File):File = { // val localFile = if(local.isDir && url.file.isDefined) local/url.file.get else local in copyTo file.out //side effect file } lazy val hrefs:List[URL] = content.pluckAll(URL.Href).toList map {url href _} } } //TODO there feels like there should be a super type of URL and File maybe something like Source?? /**TODO similar wrapper kind as File should generate Input and Output*/ class URL(val url:java.net.URL, val props:Map[String,String]=Map.empty, val postData:Option[Input]=None){ //--inter-opt def toJava:java.net.URL = url //note: loses request header props override def toString = { val header = if(props.nonEmpty) props.map{case (k,v) => s"$k:$v"}.mkString( "{", ", ", "}" ) else "" val post = if(postData.nonEmpty) " (with Post data)" else "" url.toString + header + post } def ==(that:URL):Boolean = this.toString == that.toString // def hashCode:Int = ??? //--construction def /(that:File):URL = this / that.path def /(filename:String):URL = { val basePath = path.getOrElse("") val sep = if(basePath endsWith "/") "" else "/" copy(path = Some(basePath + sep + filename)) } def /(filename:Symbol):URL = this / filename.name def *(param:(Any,Any)):URL = this ** List(param) def **(optParam:Option[(Any,Any)]):URL = this ** optParam.toList def **(moreParams:Iterable[(Any,Any)]):URL = { val newParams = moreParams.map{case (a,b) => (anyToParam(a), anyToParam(b))} def isNewKey(k:String) = newParams.exists{_._1 == k} val oldParams = params.filter{case (k,v) => !isNewKey(k)} //remove any params that exist in the new list val ps = oldParams ++ newParams if(ps.isEmpty) this else copy(query = Some(ps.map{case (a,b) => a+"="+b}.mkString("&"))) } /**expand a full url from ahref from a url*/ def href(ref:String):URL = { if(ref contains "://") URL(ref) else if(ref startsWith "/") copy(path = Some(ref)) else if(ref startsWith "#") copy(path = Some(path.getOrElse("") + ref)) else this / ref } def addHeader(kvs:Tuple2[String,String]*) = copy(props = props ++ kvs) /**quickly add tuple headers with an operator instead of [[addHeader]] */ def ^(kv:Tuple2[String,String]) = copy(props = props + kv) //--composition operator types (do the right thing) def ~(token:Token):URL = copy(props = props + ("Authorization" -> s"token ${token.value}") ) //github style token auth //TODO add other auth keys like: "Authorization: Bearer <access-token>" def ~(mime:Mime):URL = copy(props = props + ("Content-Type" -> mime.name)) def ~(param:Tuple2[Any,Any]):URL = this ** List(param) def ~(postData:Input):URL = copy(postData = Some(postData)) def ~(postFile:File):URL = this ~ postFile.in ~ postFile.mime //--support private def anyToParam(x:Any):String = x match { case Symbol(v) => v case Some(v:Any) => v.toString case v:Iterable[Any] => v map anyToParam mkString "," case _ => x.toString } // def connect(implicit ec:ExecutionContext):Future[URL.Connection] = Future(blocking{get.start}) //--simple rest // def fetch(file:File)(implicit ec:ExecutionContext):Future[File] = connect map {_ copyTo file} // def fetch(implicit ec:ExecutionContext):Future[String] = connect.map{_.in.asString} // 2017-07-30("use fetch instead since get has meaning in scala","dq") // def get(file:File)(implicit ec:ExecutionContext):Future[File] = connect map {_ copyTo file} // 2017-07-30("use fetch instead since get has meaning in scala","dq") // def get(implicit ec:ExecutionContext):Future[String] = connect.map{_.in.toString} import Implicit.ec //used as a default to make things super simple /**Ultra simple BUT blocking and many string assumptions*/ // def get:String = connect.map{_.in.asString}.block(20.s) getOrElse "" // def get[A](implicit ec:ExecutionContext, cb:cc.drx.URL.Connectable[A]):Future[A] = connect map {cb.apply} // def async[A](f:URL.Connection => A)(implicit ec:ExecutionContext):Future[A] = Future(blocking(f(get))) def async[A](implicit cb:cc.drx.URL.Connectable[A],ec:ExecutionContext):Future[A] = Future(blocking(cb(get))) // def async[A](f:URL.Connection => A):Future[A] = Future(blocking{ f(this.get) }) def get:URL.Connection = new URL.Connection(this) def in:Input = get.in ///*just use url.in copyTo outputFile **/ // def copyTo(file:File):Future[File] = connect map {_ copyTo file} def as[A](implicit cb:cc.drx.URL.Connectable[A]):A = cb apply get // def asFuture[A](implicit ec:ExecutionContext, cb:cc.drx.URL.Connectable[A]):A = connect map cb.apply /**Ultra simple BUT blocking and many many assumptions (strings return...)*/ // def post(file:File):String = (this ~ file.mime).connect(file.in).map{_.in.toString}.block(20.s) getOrElse "" // def post:String = ??? //TODO implement the form query encoded params // 2017-07-30("stop using the blocks","v0.2.15") // def get:String = connect(Implicit.ec).map{_.in.toString}.block(10.s) //forced bloc // def get(implicit ec:ExecutionContext):Future[String] = {import Implicit.ec; connect map {_.in.toString} block 20.s getOrElse ""} // def get:String = scala.io.Source.fromURL(url).mkString //TODO make this more robust with timeouts , futures and stream closing and header parsing, content type detection // def hrefs(implicit ec:ExecutionContext):Future[List[String]] = for(res <- connect map {_.in.asString}) yield { res.pluckAll(URL.Href).toList } /* recursive with futures is harder....TODO def hrefs(depth:Int)(implicit ec:ExecutionContext):Future[List[String]] = if(depth < 1 ) hrefs(ec) else for(as <- hrefs; a <- as; bs <- (this / a).hrefs(depth-1); b <- bs) yield b hrefs.map{as => as ++ {} } ; a <- as; bs <- (this / a).hrefs(depth-1); b <- bs) yield b // hrefs.map{as => as flatMap {a => (this / a) hrefs (depth-1)} } */ //--required params def protocol:String = url.getProtocol def host:String = url.getHost //--optional params def user:Option[String] = Option(url.getUserInfo) def query:Option[String] = Option(url.getQuery) def port:Option[Int] = url.getPort match {case -1 => None; case p => Option(p)} def file:Option[File] = path map File.apply def path:Option[String] = url.getPath.toOption //getPath returns empty string "" so DrxString toOption sets it to None def ref:Option[String] = Option(url.getRef) //--convenience lazy val params:List[(String,String)] = { // for(q:String <- query.toList:List[String]; p:String <- q split '&'; t:(String,String) <- (p split '=').toTuple2.toOption) yield t //TODO use build in urlencoder/decoder parser instead query.map{q => q.split('&').toList.map{p => (p split '=').toTuple2 getOrElse ("parse"->"error")} } getOrElse List() } private lazy val paramsMap = params.toMap def getParam(key:String):Option[String] = paramsMap.get(key) //--copy portions def copy( protocol:String=protocol, user:Option[String]=user, host:String=host, port:Option[Int]=port, path:Option[String]=path, query:Option[String]=query, props:Map[String,String]=props, postData:Option[Input]=postData ):URL = { val urlString:String = protocol + "://" +++ user.map{_+"@"} + host +++ port.map{":"+_} +++ path +++ query.map{"?"+_} new URL(new java.net.URL(urlString),props,postData) } }