Commit 1dc4937f authored by Mathieu's avatar Mathieu
Browse files

Implements #364

parent a7ad8d2d
Pipeline #366 passed with stage
in 22 minutes and 14 seconds
......@@ -432,3 +432,9 @@ lazy val gridscaleWebDAV = OsgiProject(dir, "gridscale.webdav", imports = Seq("*
version := gridscaleVersion
) settings(settings: _*) dependsOn(gridscale, gridscaleHTTP)
lazy val xzJava = OsgiProject(dir, "xzjava", imports = Seq("*"), exports = Seq("org.tukaani.*")) settings (
libraryDependencies += "org.tukaani" % "xz" % "1.8",
version := "1.8"
) settings(settings: _*)
......@@ -87,7 +87,7 @@ def allThirdParties = Seq(
txtmark)
lazy val openmoleCache = OsgiProject(thirdPartiesDir, "org.openmole.tool.cache", imports = Seq("*")) dependsOn (openmoleLogger) settings (thirdPartiesSettings: _*) settings (libraryDependencies += Libraries.squants, libraryDependencies += Libraries.cats)
lazy val openmoleTar = OsgiProject(thirdPartiesDir, "org.openmole.tool.tar", imports = Seq("*")) dependsOn (openmoleFile) settings (thirdPartiesSettings: _*)
lazy val openmoleTar = OsgiProject(thirdPartiesDir, "org.openmole.tool.tar", imports = Seq("*")) dependsOn (openmoleFile) settings (thirdPartiesSettings: _*) settings(libraryDependencies += Libraries.xzJava)
lazy val openmoleFile = OsgiProject(thirdPartiesDir, "org.openmole.tool.file", imports = Seq("*")) dependsOn(openmoleLock, openmoleStream, openmoleLogger) settings (thirdPartiesSettings: _*)
lazy val openmoleLock = OsgiProject(thirdPartiesDir, "org.openmole.tool.lock", imports = Seq("*")) settings (thirdPartiesSettings: _*)
lazy val openmoleLogger = OsgiProject(thirdPartiesDir, "org.openmole.tool.logger", imports = Seq("*")) dependsOn(openmoleOutputRedirection) settings (thirdPartiesSettings: _*) settings (libraryDependencies += Libraries.sourceCode)
......
......@@ -232,7 +232,7 @@ class FileToolBox(initSafePath: SafePath) {
div(omsheet.centerElement)(
downloadTrigger,
DataUtils.fileToExtension(initSafePath.name) match {
case FileExtension.TGZ | FileExtension.TAR | FileExtension.ZIP
case FileExtension.TGZ | FileExtension.TAR | FileExtension.ZIP | FileExtension.TXZ
iconAction(fileaction.extract, archive, "extract")
case _ span
},
......
......@@ -141,6 +141,10 @@ package object data {
def displayable = false
}
object TarXz extends FileExtension {
def displayable = false
}
object Tar extends FileExtension {
def displayable = false
}
......@@ -170,6 +174,7 @@ package object data {
val NO_EXTENSION = EditableFile("text")
val SVG = SVGExtension
val TGZ = TarGz
val TXZ = TarXz
val TAR = Tar
val ZIP = Zip
val TGZBIN = TgzBin
......@@ -505,7 +510,7 @@ package object data {
else if (fileName.endsWith("nlogo")) CodeFile(NetLogoLanguage())
else if (fileName.endsWith("R")) CodeFile(RLanguage())
else if (fileName.endsWith("jar")) JarArchive
else if (fileName.endsWith("tgz") || fileName.endsWith("tar.gz")) Archive
else if (fileName.endsWith("tgz") || fileName.endsWith("tar.gz") || fileName.endsWith("tar.xz")) Archive
else UndefinedFileType
}
......@@ -826,8 +831,10 @@ package object data {
case class SequenceData(header: SequenceHeader = Seq(), content: Seq[Array[String]] = Seq()) {
def withRowIndexes = {
val lineIndexes = (1 to content.length).map { _.toString }
def withRowIndexes = {
val lineIndexes = (1 to content.length).map {
_.toString
}
copy(header = header :+ "Row index", content = content.zip(lineIndexes).map { case (l, i) l :+ i })
}
}
......
......@@ -51,24 +51,27 @@ object DataUtils {
IOArgs(jarMethod.argumentTypes, typeStringToVariableElement(jarMethod.returnType, jarMethod.argumentTypes.size))
}
implicit def fileToExtension(fileName: String): FileExtension = fileName match {
case x if x.endsWith(".oms") OMS
case x if x.endsWith(".csv") CSV
case x if x.endsWith(".nlogo") | x.endsWith(".nlogo3d") NETLOGO
case x if x.endsWith(".R") R
case x if x.endsWith(".gaml") |
x.endsWith(".py") |
x.endsWith(".txt") | x.endsWith(".nls") TEXT
case x if x.endsWith(".md") MD
case x if x.endsWith(".tgz") | x.endsWith(".tar.gz") TGZ
case x if x.endsWith(".tar") TAR
case x if x.endsWith(".zip") ZIP
case x if x.endsWith(".tgz.bin") | x.endsWith(".tar.gz.bin") TGZBIN
case x if x.endsWith(".jar") JAR
case x if x.endsWith(".scala") SCALA
case x if x.endsWith(".sh") SH
case x if x.endsWith(".svg") SVG
case _ BINARY
implicit def fileToExtension(fileName: String): FileExtension = {
fileName match {
case x if x.endsWith(".oms") OMS
case x if x.endsWith(".csv") CSV
case x if x.endsWith(".nlogo") | x.endsWith(".nlogo3d") NETLOGO
case x if x.endsWith(".R") R
case x if x.endsWith(".gaml") |
x.endsWith(".py") |
x.endsWith(".txt") | x.endsWith(".nls") TEXT
case x if x.endsWith(".md") MD
case x if x.endsWith(".tgz") | x.endsWith(".tar.gz") TGZ
case x if x.endsWith(".tar.xz") TXZ
case x if x.endsWith(".tar") TAR
case x if x.endsWith(".zip") ZIP
case x if x.endsWith(".tgz.bin") | x.endsWith(".tar.gz.bin") TGZBIN
case x if x.endsWith(".jar") JAR
case x if x.endsWith(".scala") SCALA
case x if x.endsWith(".sh") SH
case x if x.endsWith(".svg") SVG
case _ BINARY
}
}
implicit class CleanName(s: String) {
......
......@@ -151,7 +151,10 @@ class ApiImpl(s: Services, applicationControl: ApplicationControl) extends Api {
from.extractUncompress(to, true)
to.applyRecursive((f: File) f.setWritable(true))
case Zip Utils.unzip(from, to)
case _ throw new Throwable("Unknown compression format for " + from.getName)
case TarXz
from.extractUncompressXZ(to, true)
to.applyRecursive((f: File) f.setWritable(true))
case _ throw new Throwable("Unknown compression format for " + from.getName)
}
} match {
case Success(_) ExtractResult.ok
......@@ -161,7 +164,7 @@ class ApiImpl(s: Services, applicationControl: ApplicationControl) extends Api {
def extractTGZ(safePath: SafePath): ExtractResult = {
DataUtils.fileToExtension(safePath.name) match {
case FileExtension.TGZ | FileExtension.TAR | FileExtension.ZIP
case FileExtension.TGZ | FileExtension.TAR | FileExtension.ZIP | FileExtension.TXZ
val archiveFile = safePathToFile(safePath)(ServerFileSystemContext.project, workspace)
val toFile: File = safePathToFile(safePath.parent)(ServerFileSystemContext.project, workspace)
extractArchiveFromFiles(archiveFile, toFile)(ServerFileSystemContext.project)
......@@ -336,7 +339,13 @@ class ApiImpl(s: Services, applicationControl: ApplicationControl) extends Api {
t match {
case ce: ScalaREPL.CompilationError
def toErrorWithLocation(em: ScalaREPL.ErrorMessage) =
ErrorWithLocation(em.rawMessage, em.position.map { _.line }, em.position.map { _.start }, em.position.map { _.end })
ErrorWithLocation(em.rawMessage, em.position.map {
_.line
}, em.position.map {
_.start
}, em.position.map {
_.end
})
ErrorData(ce.errorMessages.map(toErrorWithLocation), t)
case _ ErrorData(t)
......@@ -353,14 +362,18 @@ class ApiImpl(s: Services, applicationControl: ApplicationControl) extends Api {
case ErrorInCode(e) Some(error(e))
case ErrorInCompiler(e) Some(error(e))
case compiled: Compiled
onCompiled.foreach { _(execId) }
onCompiled.foreach {
_(execId)
}
catchAll(OutputManager.withStreamOutputs(outputStream, outputStream)(compiled.eval)) match {
case Failure(e) Some(error(e))
case Success(dsl)
val services = MoleServices.copy(MoleServices.create)(outputRedirection = OutputRedirection(outputStream))
Try(dslToPuzzle(dsl).toExecution()(services)) match {
case Success(ex)
onEvaluated.foreach { _(ex, execId) }
onEvaluated.foreach {
_(ex, execId)
}
None
case Failure(e) Some(error(e))
}
......@@ -536,6 +549,7 @@ class ApiImpl(s: Services, applicationControl: ApplicationControl) extends Api {
gridscale.http.getResponse(checkedURL) { response
def extractName = checkedURL.split("/").last
val name =
response.headers.flatMap {
case ("Content-Disposition", value)
......
......@@ -103,6 +103,7 @@ object Libraries {
lazy val sourceCode = "org.openmole.library" %% "sourcecode" % sourcecodeVersion
lazy val txtmark = "org.openmole.library" %% "com-github-rjeschke-txtmark" % "0.13"
lazy val spatialdata = "org.openmole.library" %% "org-openmole-spatialdata" % spatialdataVersion
lazy val xzJava = "org.openmole.library" %% "xzjava" % "1.8"
def httpClientVersion = "4.5.3"
lazy val httpClient =
......
......@@ -16,11 +16,14 @@
*/
package org.openmole.tool
import java.io.{ IOException, File }
import java.io._
import java.nio.file._
import org.openmole.tool.file._
import org.openmole.tool.stream._
import scala.collection.mutable.{ Stack, ListBuffer }
import org.tukaani.xz.{ LZMA2Options, XZInputStream, XZOutputStream }
import scala.collection.mutable.{ ListBuffer, Stack }
import scala.collection.JavaConverters._
package object tar {
......@@ -87,6 +90,48 @@ package object tar {
}
}
implicit class XZFileDecorator(file: File) {
def extractXZ(to: File) = {
if (!file.getName.endsWith(".xz")) throw new java.io.IOException(s"$file is not a XZ file")
else {
val inputStream = new FileInputStream(file)
val outputStream = new FileOutputStream(to)
val inxz = extractToStream
val buffer = new Array[Byte](inputStream.available)
Iterator.continually(inxz.read(buffer)).takeWhile(_ != -1).foreach {
outputStream.write(buffer, 0, _)
}
inxz.close
}
}
def extractToStream: InputStream = {
if (!file.getName.endsWith(".xz")) throw new java.io.IOException(s"$file is not a XZ file")
else {
val inputStream = new FileInputStream(file)
new XZInputStream(inputStream, 100 * 1024)
}
}
def compressXZ(to: File) = {
val outfile = new FileOutputStream(to)
val outxz = new XZOutputStream(outfile, new LZMA2Options(8), org.tukaani.xz.XZ.CHECK_SHA256)
val infile = new FileInputStream(file)
val buffer = new Array[Byte](8192)
Iterator.continually(infile.read(buffer)).takeWhile(_ != -1).foreach { size
outxz.write(buffer, 0, size)
}
outxz.finish
}
}
implicit class FileTarArchiveDecorator(file: File) {
def archive(dest: File, time: Boolean = true) =
......@@ -110,6 +155,12 @@ package object tar {
_.extract(dest, overwrite)
}
def extractUncompressXZ(dest: File, overwrite: Boolean = false) = {
withClosable(new TarInputStream(file.extractToStream)) {
_.extract(dest, overwrite)
}
}
def copyCompress(toF: File): File = {
if (toF.isDirectory) file.archiveCompress(toF)
else file.copyCompressFile(toF)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment