/* Copyright 2010 Aaron J. Radke Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cc.drx /**Explicit repository Source(file,url) and Style(maven,ivy,sbt)*/ class Repo(val source:Repo.Source, val name:Option[String], val style:Repo.Style){ import Repo._ def releases(x:Project) :List[Release] = source.releases(x, style) def projects(x:Org) :List[Project] = source.projects(x, style) def artifacts(x:Release):List[Artifact] = source.artifacts(x, style) // def latest(x:Project):Option[Release] = releases(x).sortBy{_.tag}.lastOption // def latest(org:String,prj:String):Option[Release] = releases(x).sortBy{_.tag}.lastOption def get(a:Artifact):Option[File] = source.get(a,style) def as(newStyle:Style):Repo = new Repo(source,name,newStyle) def /(orgName:String):Org = Org(orgName) override def toString = s"Repo($source, $name, $style)" } //TODO remove extra toList functions //TODO add auto detect of file type urls and use file.list instead of webpage grok //TODO add filter of artifact types //TODO generalize organization and artifacts for other layouts like ivy.. /** (Repo) can find (Org -> Project -> Release -> Artifact) * Repo have two major input types: * 1) user reference using org/project/release * 2) dir listing of an artifact path (whose parents or filename can determine the org/project/release * * function styles end with F */ object Repo{ /** Org -> Project -> Release -> Artifact */ case class Artifact(release:Release, ext:String, alt:Option[String]=None){ def *(newAlt:String):Artifact = copy(alt=Some(newAlt)) def alt(newAlt:String):Artifact = copy(alt=Some(newAlt)) //TODO def get:Try[File] = ??? is the file already cached?? else download def path(implicit style:Style):File = style.path(this) def baseKind:String = alt match { case Some("javadoc") => "doc" case Some("sources") => "src" case _ => ext.split(".").head } } private def clean(n:String):String = { val a = if(n endsWith "/") n dropRight 1 else n val b = if(a startsWith ":") a drop 1 else a return b } /** Org -> Project -> Release */ object Release{ def apply(org:String, prj:String, ver:String):Release = Org(org)/prj/ver } case class Release(project:Project, tag:Version){ private def baseAndAlt = tag.name.span(_ != '-') def base:String = Version(baseAndAlt._1).name def alt:Option[String] = {val a = baseAndAlt._1; if(a == "") None else Some(a)} def *(altNew:String) = Release(project, Version(base +++ alt.map{"-"+_})) private val ArtifactAltPat = """\-([^.]+)\.(.+)""".r private val ArtifactPat = """\.(.+)""".r def /(ext:String):Artifact = { //TODO use the style decomposers instead of these custom tags val inits = List( project.name+"-"+tag.name, project.name +++ project.alt.map("_"+_) ) inits find {ext startsWith _} map { init => val rest = ext.drop(init.size) rest match { case ArtifactAltPat(alt,ext) => Artifact(this,ext,Some(alt)) case ArtifactPat(ext) => Artifact(this,ext) case _ => Artifact(this,"",Some(rest)) } } getOrElse Artifact(this,ext) } def list(implicit repos:Repos) = listFirst(repos){_ artifacts this} def path(implicit style:Style):File = style.path(this) def nice = project.org.name .fit(40,cc.drx.Style.Left) + "# " + project.name .fit(20,cc.drx.Style.Left) + ": " + tag.name } /** Org -> Project */ case class Project(org:Org, name:String, alt:Option[String]=None){ def *(altNew:String) = copy(alt=Some(altNew)) def /(tag:String) = Release(this, Version(clean(tag))) def list(implicit repos:Repos):List[Release] = listFirst(repos){_ releases this}.toList sortBy (_.tag) def latest(implicit repos:Repos):Option[Release] = list(repos).lastOption def stable(implicit repos:Repos):Option[Release] = list(repos).filter(_.tag.isFinal).lastOption def path(implicit style:Style):File = style.path(this) def stats(implicit repos:Repos):Map[String,Int] = repos.headOption.map{_.source.stats(this)} getOrElse Map() } private val ScalaProjectAltPat = """([^_]+)_(.+)""".r /** Org */ object Org{ def apply(org:String):Org = Org(org.trim.split("""[./]+""").toList) } case class Org(domains:List[String], alt:Option[String]=None){ def name = domains mkString "." /**sbt style autodetect alt scala version style*/ def %%(prj:String) = clean(prj) match { case ScalaProjectAltPat(p,alt) => Project(this, p, Some(alt)) case p => Project(this, p, None) } /**sbt style alias for '/' */ def %(prj:String) = this / prj def /(prj:String) = Project(this, clean(prj), None) def *(altNew:String) = copy(alt=Some(altNew)) def list(implicit repos:Repos) = listFirst(repos){_.projects(this).toList} //TODO add find first rather than find all def path(implicit style:Style):File = style.path(this) } //implementation functions // private def hrefs(url:URL,depth:Int=0):Future[List[String]] = { private def files(file:File,depth:Int=0):List[String] = { val names = if(depth == 0) file.list else file.walk(maxDepth=depth) filter (_.isFile) names.toList map {_.name} filterNot blackList } private def hrefs(url:URL):List[String] = url.get.hrefs.flatMap{_.file}.map{_.name} filterNot blackList private val blackStarts = ". # maven-metadata readme" split " " private def blackList(name:String):Boolean = blackStarts exists (name.toLowerCase startsWith _) //-----Styles-------------- object Style{ implicit val defaultStyle:Style = Maven } trait Style{ def filename(o:Org):String def filename(p:Project):String def filename(r:Release):String def filename(a:Artifact):String //TODO make this abstract and implement specifically for each repo style //This provides a mechanism from a listing to a canonical form def artifact(f:File):Option[Artifact] = for(pf <- f.parent; r <- release(pf) ) yield r / f.name def release(f:File):Option[Release] = for(pf <- f.parent; p <- project(pf) ) yield p / f.name def project(f:File):Option[Project] = for(pf <- f.parent; o <- org(pf) ) yield o / f.name def org(f:File):Option[Org] = Some(Org(f.unixPath)) def artifactDepth:Int def releaseDepth:Int = 0 def projectDepth:Int = 0 //---- def path(o:Org):File = File(filename(o)) def path(p:Project):File = path(p.org) / filename(p) def path(r:Release):File = path(r.project) / filename(r) def path(a:Artifact):File = path(a.release) / filename(a) } /** <o/r/g> / <prj>[_alt] / <tag> / <prj>[_alt]-<tag>.<ext> */ object Maven extends Style{ def artifactDepth:Int = 0 def filename(o:Org):String = o.domains mkString "/" def filename(p:Project):String = p.name +++ p.alt.map("_"+_) def filename(r:Release):String = r.tag.name def filename(a:Artifact):String = filename(a.release.project) + ("-"+a.release.tag.name) +++ a.alt.map("-"+_) + ("."+a.ext) } /** <org> / <prj>[_alt] / <tag> / <ext>s / <prj>[_alt].<ext> */ object Ivy extends Style{ def artifactDepth:Int = 1 def filename(o:Org):String = o.domains mkString "." def filename(p:Project):String = p.name +++ p.alt.map("_"+_) def filename(r:Release):String = r.tag.name def filename(a:Artifact):String = { val baseDir = a.baseKind+"s/" if(a.baseKind == "ivy") baseDir + a.ext else baseDir + filename(a.release.project) + "."+a.ext } /** copy (& transform) a cache or repo (useful for maintaining repos on air-gapped networks) * Example: Repo.Ivy.sync(file"~/.ivy/cache", file"~/ivyrepo", dryRun=false) */ lazy val sync = Sync(src => { val ivyXml = Glob.ignoreCase("ivy-*.xml") val ivyData = Glob.ignoreCase("ivydata-*.properties") val ivyOrg = Glob.ignoreCase("ivy-*.xml.original") def parent = src.parent.getOrElse(src) //simple way to do parent matches val name = src.name if(name == ".sbt.ivy.lock") Sync.Skip(src, Symbol("SbtLock")) else if(ivyOrg matches name) Sync.Skip(src, Symbol("IvyOrg")) else if(ivyData matches name) Sync.Skip(src, Symbol("IvyData")) else if(src.path contains "-SNAPSHOT") Sync.Skip(src, Symbol("Snapshot")) else if( (ivyXml matches name) && (parent.name != "ivys")) { Sync.Copy( File(src.path.replace(name, s"ivys/$name")), Symbol("IvyXml")) //transform the location } else Sync.Copy(src, Symbol("Artifact")) }) } /** <org> / <prj>[_alt] / <tag> / <ext>s / <prj>[_alt].<ext> */ object IvyCache extends Style{ def artifactDepth:Int = 1 def filename(o:Org):String = o.domains mkString "." def filename(p:Project):String = p.name +++ p.alt.map("_"+_) def filename(r:Release):String = "" //no specific release folder def filename(a:Artifact):String = if(a.baseKind == "ivy") "ivy-"+a.release.tag.name+ a.ext.drop(3) //drop ivy from the ext else a.baseKind+"s/" + filename(a.release.project) + "-" + a.release.tag.name + "."+a.ext } object GithubStyle extends Style{ def artifactDepth:Int = 0 def filename(o:Org):String = ??? def filename(p:Project):String = ??? def filename(r:Release):String = ??? def filename(a:Artifact):String = ??? } /** <org> / <prj> / scala_<alt> / sbt_<alt> / <tag> / <ext>s / <prj>.<ext> */ case class SbtPlugin(sbtVersion:String="0.13",scalaVersion:String="2.10") extends Style{ //TODO use implicits for defaults of sbtVersion and scalaVersion //TODO test each subsection def artifactDepth:Int = 1 def filename(o:Org):String = o.domains mkString "." def filename(p:Project):String = p.name +++ p.alt.map("_"+_) + s"/scala_$scalaVersion/sbt_$sbtVersion" def filename(r:Release):String = r.tag.name def filename(a:Artifact):String = a.ext+"s/" + a.release.project.name + "."+a.ext } //-----Sources-------------- /**Generic repo source (required functions)*/ trait Source { //--required def get(a:Artifact,style:Style):Option[File] def releases(x:Project,style:Style):List[Release] def projects(x:Org, style:Style):List[Project] def artifacts(x:Release, style:Style):List[Artifact] def stats(p:Project):Map[String,Int] = Map[String,Int]() } trait SourcePathList extends Source{ def list(f:File,depth:Int):List[String] //---- def releases(x:Project,style:Style):List[Release] = list(style path x, style.releaseDepth) map (x / _ ) def projects(x:Org, style:Style):List[Project] = list(style path x, style.projectDepth) map (x / _ ) def artifacts(x:Release, style:Style):List[Artifact] = list(style path x, style.artifactDepth) map (x / _ ) } class SourceFile(baseDir:File) extends SourcePathList { def list(f:File,depth:Int):List[String] = files(baseDir / f.unixPath, depth).toList def get(a:Artifact, style:Style):Option[File] = {val f = style.path(a); if(f.exists) Some(f) else None} } class SourceURL(baseURL:URL) extends SourcePathList { /**note: depth is ignored in url souces*/ def list(f:File,depth:Int):List[String] = hrefs(baseURL / (f.unixPath + "/") ) def get(a:Artifact,style:Style):Option[File] = ??? //TODO add url downloader to a cache repo } //TODO move most of this functionality and specific parsing to the github object itself so it can be used independently class SourceGithub(token:Option[String] = None)(implicit ec:ExecutionContext=Implicit.ec) extends Source { //api ex: https://api.github.com/repos/scala/scala/tags private val api = { val url = ( URL(s"https://api.github.com") ** List("per_page" -> "100", "page"-> "1") ).addHeader("Accept" -> "application/vnd.github.v3+json") token.map{t => url.addHeader("Authorization" -> s"token $t") } getOrElse url } override def toString = s"SourceGithub($api)" //TODO add a per_page=100 parameter for better pagination https://developer.github.com/guides/traversing-with-pagination/ //TODO get next from the rel.next header to list the with per_page=100 parameter for better pagination https://developer.github.com/guides/traversing-with-pagination/ private val Name = Pluck(""""name"\s*:\s*"([^"]+)"""".r) private val FullName = Pluck(""""full_name"\s*:\s*"([^"]+)"""".r) private val Title = Pluck(""""title"\s*:\s*"([^"]+)"""".r) //top-level title private val State = Pluck(""""state"\s*:\s*"([^"]+)"""".r) //top-level title private val Number = Pluck(""""number"\s*:\s*(\d+)""".r) //top-level title private val UserType = Pluck(""""type"\s*:\s*"([^"]+)"""".r) // 2017-07-30("use future wrapped version","dq") // private def blockingGet(url:URL):String = { // url.fetch.block(60.s).getOrElse(s"Error downloading $url") // } //--blocking methods to meet the Source contract def releases(x:Project,style:Style):List[Release] = releasesF(x).block(2.minute).getOrElse(Nil) def projects(org:Org, style:Style):List[Project] = projectsF(org).block(2.minute).getOrElse(Nil) //--future based methods def releasesF(x:Project):Future[List[Release]] = (api / "repos" / x.org.name / x.name / "tags").async{ c => c.content.pluckAll(Name).toList map {x / _} } //--cache for user type (org or user) private val isOrgType = TrieMap[String,Future[Boolean]]().withDefault{(name:String) => (api / "users" / name).async{c => c.content.pluckAll(UserType).toList.exists{_ == "Organization" } } } private val pageLimit = 99 private def mapPagesF[A](query:URL)(f: String => List[A]):Future[List[A]] = { def getF(acc:List[A], page:Int):Future[List[A]] = { val ffs = (query * ("page" -> page.toString)).async{c => f(c.content)} // val lastPage = c.relLinks.get("last").flatMap{url => url getParam "page" map {_.toInt}}.getOrElse(page) ffs.flatMap{fs => if(fs.size < 100 || page > pageLimit) Future(acc ++ fs) //pageLimit set to 99 //note this uses stack to recurse.. else getF(acc ++ fs, page + 1) } } getF(Nil, 1) //start from the first page } def projectsF(org:Org):Future[List[Project]] = { isOrgType(org.name).flatMap{ (orgType:Boolean) => val orgKey = if (orgType) "orgs" else "users" val query = api / orgKey / org.name / "repos" mapPagesF(query){resp => resp.pluckAll(FullName).toList.map{fullName => val prjName = fullName.replace(org.name+"/", "") //take off the organization name if it exists org / prjName }:List[Project] } } } def issuesF(prj:Project):Future[List[Github.Issue]] = { val query = api / "repos" / prj.org.name / prj.name / "issues" mapPagesF(query){resp => val json = Json(resp) def issueSM(json:Json) = StringMap.collect{ case "labels" => (json getSeq "labels").toList map {labels => labels flatMap {_ getString "name"} } mkString "," } mergeWith json for( j <- json.asSeq.toList; body <- j getString "body"; number <- j getInt "number"; //this is an Int not a Double state <- j getString "state"; title <- j getString "title"; labels <- j getSeq "labels" map {labels => labels flatMap {_ getString "name"} }; // milestone <- j get "milestone" flatMap {_ getString "title"}; // j getBoolean "locked" comments <- j getInt "comments"; //this is an Int count // j getString "closed_at") //these are dates // j getString "created_at") updated_at <- j getString "updated_at" // println(issue getMap "pull_request" flatMap {_ get "url"}) // println(issue getMap "repository" flatMap {_ get "full_name"} ) ) yield { // TODO include assignees back // val assignee = j get "assignee" flatMap {_ getString "login"} // val assignees = j getSeq "assignees" map {as => as flatMap {_ getString "login"} }; Github.Issue( body = body, number = number, state = state, title = title, // labels = labels, //TODO // assignees = Seq(), //assignees :+ assignee, //TODO // milestone = milestone, //TODO comments = comments, updated_at = Date(updated_at) ) } } } def artifacts(x:Release, style:Style):List[Artifact] = List(x / "zip") //TODO make the get actually get this zipball def get(a:Artifact,style:Style):Option[File] = ??? //TODO add url downloader to a cache repo def statsF(p:Project):Future[Map[String,Int]] = { (api / "repos" / p.org.name / p.name).async{c => val json = c.content def count(k:String):Option[Int] = json pluck Pluck((k.quote + """\s*:\s*(\d+)""").r) map {_.toInt} val keys = "forks watchers stargazers open_issues subscribers" split " " (for(k <- keys; v <- count(k+"_count")) yield k->v).toMap } } override def stats(p:Project):Map[String,Int] = statsF(p).block(60.s).getOrElse(Map.empty) // val json = blockingGet(api / "repos" / p.org.name / p.name) // def count(k:String):Option[Int] = json pluck Pluck((k.quote + """\s*:\s*(\d+)""").r) map {_.toInt} // (for(k <- "forks watchers stargazers open_issues subscribers" split " "; v <- count(k+"_count")) yield k->v).toMap // // "forks watchers stargazers open_issues subscribers".split(" ") mapIf; v <- count(k+"_count")) yield k->v).toMap //TODO make this cleaner with a mapIf that flatten's options // } } private def listFirst[B](repos:Repos)(list:Repo => List[B]):List[B] = repos.foldLeft(List[B]()){ case (Nil, repo) => list(repo) case (x, _) => x } def apply(base:File, style:Style):Repo = new Repo(new SourceFile(base), None, style) def apply(base:URL, style:Style):Repo = new Repo(new SourceURL(base), None, style) //---repos //val github = ??? val ivyLocal = Repo(File.home / ".ivy2/local", Ivy) val ivyCache = Repo(File.home / ".ivy2/cache", IvyCache) val mavenLocal = Repo(File.home / ".m2/repository", Maven) val mavenCentral = Repo(URL("https://repo.maven.apache.org/maven2"), Maven) //http://dl.bintray.com/sbt/sbt-plugin-releases/com.eed3si9n/sbt-assembly/scala_2.10/sbt_0.13/ // val mavenCentralGoogle = Repo(url"https://maven-central.storage.googleapis.com",Maven) val sbtPlugins = Repo(URL("http://dl.bintray.com/sbt/sbt-plugin-releases"), SbtPlugin()) lazy val github = new Repo(new SourceGithub(OS.Env.get("GITHUB_TOKEN")), None, GithubStyle) def github(token:String) = new Repo(new SourceGithub(Some(token)), None, GithubStyle) type Repos = Iterable[Repo] implicit val defaultRepos:Repos = List[Repo](ivyLocal, ivyCache, mavenLocal, mavenCentral, sbtPlugins, github) }