Skip to content

GitLab

Commit 1dc4937f authored by Mathieu's avatar Mathieu
Browse files

Implements #364

parent a7ad8d2d
Pipeline #366 passed with stage
in 22 minutes and 14 seconds
Hide whitespace changes
Inline Side-by-side
libraries/build.sbt
View file @ 1dc4937f
......@@ -432,3 +432,9 @@ lazy val gridscaleWebDAV = OsgiProject(dir, "gridscale.webdav", imports = Seq("*
version := gridscaleVersion
) settings(settings: _*) dependsOn(gridscale, gridscaleHTTP)
lazy val xzJava = OsgiProject(dir, "xzjava", imports = Seq("*"), exports = Seq("org.tukaani.*")) settings (
libraryDependencies += "org.tukaani" % "xz" % "1.8",
version := "1.8"
) settings(settings: _*)
openmole/build.sbt
View file @ 1dc4937f
......@@ -87,7 +87,7 @@ def allThirdParties = Seq(
txtmark)
lazy val openmoleCache = OsgiProject(thirdPartiesDir, "org.openmole.tool.cache", imports = Seq("*")) dependsOn (openmoleLogger) settings (thirdPartiesSettings: _*) settings (libraryDependencies += Libraries.squants, libraryDependencies += Libraries.cats)
lazy val openmoleTar = OsgiProject(thirdPartiesDir, "org.openmole.tool.tar", imports = Seq("*")) dependsOn (openmoleFile) settings (thirdPartiesSettings: _*)
lazy val openmoleTar = OsgiProject(thirdPartiesDir, "org.openmole.tool.tar", imports = Seq("*")) dependsOn (openmoleFile) settings (thirdPartiesSettings: _*) settings(libraryDependencies += Libraries.xzJava)
lazy val openmoleFile = OsgiProject(thirdPartiesDir, "org.openmole.tool.file", imports = Seq("*")) dependsOn(openmoleLock, openmoleStream, openmoleLogger) settings (thirdPartiesSettings: _*)
lazy val openmoleLock = OsgiProject(thirdPartiesDir, "org.openmole.tool.lock", imports = Seq("*")) settings (thirdPartiesSettings: _*)
lazy val openmoleLogger = OsgiProject(thirdPartiesDir, "org.openmole.tool.logger", imports = Seq("*")) dependsOn(openmoleOutputRedirection) settings (thirdPartiesSettings: _*) settings (libraryDependencies += Libraries.sourceCode)
......
openmole/gui/client/org.openmole.gui.client.core/src/main/scala/org/openmole/gui/client/core/files/FileToolBox.scala
View file @ 1dc4937f
......@@ -232,7 +232,7 @@ class FileToolBox(initSafePath: SafePath) {
div(omsheet.centerElement)(
downloadTrigger,
DataUtils.fileToExtension(initSafePath.name) match {
case FileExtension.TGZ | FileExtension.TAR | FileExtension.ZIP ⇒
case FileExtension.TGZ | FileExtension.TAR | FileExtension.ZIP | FileExtension.TXZ ⇒
iconAction(fileaction.extract, archive, "extract")
case _ ⇒ span
},
......
openmole/gui/ext/org.openmole.gui.ext.data/src/main/scala/org/openmole/gui/ext/data/Data.scala
View file @ 1dc4937f
......@@ -141,6 +141,10 @@ package object data {
def displayable = false
}
object TarXz extends FileExtension {
def displayable = false
}
object Tar extends FileExtension {
def displayable = false
}
......@@ -170,6 +174,7 @@ package object data {
val NO_EXTENSION = EditableFile("text")
val SVG = SVGExtension
val TGZ = TarGz
val TXZ = TarXz
val TAR = Tar
val ZIP = Zip
val TGZBIN = TgzBin
......@@ -505,7 +510,7 @@ package object data {
else if (fileName.endsWith("nlogo")) CodeFile(NetLogoLanguage())
else if (fileName.endsWith("R")) CodeFile(RLanguage())
else if (fileName.endsWith("jar")) JarArchive
else if (fileName.endsWith("tgz") || fileName.endsWith("tar.gz")) Archive
else if (fileName.endsWith("tgz") || fileName.endsWith("tar.gz") || fileName.endsWith("tar.xz")) Archive
else UndefinedFileType
}
......@@ -826,8 +831,10 @@ package object data {
case class SequenceData(header: SequenceHeader = Seq(), content: Seq[Array[String]] = Seq()) {
def withRowIndexes = {
val lineIndexes = (1 to content.length).map { _.toString }
def withRowIndexes = {
val lineIndexes = (1 to content.length).map {
_.toString
}
copy(header = header :+ "Row index", content = content.zip(lineIndexes).map { case (l, i) ⇒ l :+ i })
}
}
......
openmole/gui/ext/org.openmole.gui.ext.data/src/main/scala/org/openmole/gui/ext/data/DataUtils.scala
View file @ 1dc4937f
......@@ -51,24 +51,27 @@ object DataUtils {
IOArgs(jarMethod.argumentTypes, typeStringToVariableElement(jarMethod.returnType, jarMethod.argumentTypes.size))
}
implicit def fileToExtension(fileName: String): FileExtension = fileName match {
case x if x.endsWith(".oms") ⇒ OMS
case x if x.endsWith(".csv") ⇒ CSV
case x if x.endsWith(".nlogo") | x.endsWith(".nlogo3d") ⇒ NETLOGO
case x if x.endsWith(".R") ⇒ R
case x if x.endsWith(".gaml") |
x.endsWith(".py") |
x.endsWith(".txt") | x.endsWith(".nls") ⇒ TEXT
case x if x.endsWith(".md") ⇒ MD
case x if x.endsWith(".tgz") | x.endsWith(".tar.gz") ⇒ TGZ
case x if x.endsWith(".tar") ⇒ TAR
case x if x.endsWith(".zip") ⇒ ZIP
case x if x.endsWith(".tgz.bin") | x.endsWith(".tar.gz.bin") ⇒ TGZBIN
case x if x.endsWith(".jar") ⇒ JAR
case x if x.endsWith(".scala") ⇒ SCALA
case x if x.endsWith(".sh") ⇒ SH
case x if x.endsWith(".svg") ⇒ SVG
case _ ⇒ BINARY
implicit def fileToExtension(fileName: String): FileExtension = {
fileName match {
case x if x.endsWith(".oms") ⇒ OMS
case x if x.endsWith(".csv") ⇒ CSV
case x if x.endsWith(".nlogo") | x.endsWith(".nlogo3d") ⇒ NETLOGO
case x if x.endsWith(".R") ⇒ R
case x if x.endsWith(".gaml") |
x.endsWith(".py") |
x.endsWith(".txt") | x.endsWith(".nls") ⇒ TEXT
case x if x.endsWith(".md") ⇒ MD
case x if x.endsWith(".tgz") | x.endsWith(".tar.gz") ⇒ TGZ
case x if x.endsWith(".tar.xz") ⇒ TXZ
case x if x.endsWith(".tar") ⇒ TAR
case x if x.endsWith(".zip") ⇒ ZIP
case x if x.endsWith(".tgz.bin") | x.endsWith(".tar.gz.bin") ⇒ TGZBIN
case x if x.endsWith(".jar") ⇒ JAR
case x if x.endsWith(".scala") ⇒ SCALA
case x if x.endsWith(".sh") ⇒ SH
case x if x.endsWith(".svg") ⇒ SVG
case _ ⇒ BINARY
}
}
implicit class CleanName(s: String) {
......
openmole/gui/server/org.openmole.gui.server.core/src/main/scala/org/openmole/gui/server/core/ApiImpl.scala
View file @ 1dc4937f
......@@ -151,7 +151,10 @@ class ApiImpl(s: Services, applicationControl: ApplicationControl) extends Api {
from.extractUncompress(to, true)
to.applyRecursive((f: File) ⇒ f.setWritable(true))
case Zip ⇒ Utils.unzip(from, to)
case _ ⇒ throw new Throwable("Unknown compression format for " + from.getName)
case TarXz ⇒
from.extractUncompressXZ(to, true)
to.applyRecursive((f: File) ⇒ f.setWritable(true))
case _ ⇒ throw new Throwable("Unknown compression format for " + from.getName)
}
} match {
case Success(_) ⇒ ExtractResult.ok
......@@ -161,7 +164,7 @@ class ApiImpl(s: Services, applicationControl: ApplicationControl) extends Api {
def extractTGZ(safePath: SafePath): ExtractResult = {
DataUtils.fileToExtension(safePath.name) match {
case FileExtension.TGZ | FileExtension.TAR | FileExtension.ZIP ⇒
case FileExtension.TGZ | FileExtension.TAR | FileExtension.ZIP | FileExtension.TXZ ⇒
val archiveFile = safePathToFile(safePath)(ServerFileSystemContext.project, workspace)
val toFile: File = safePathToFile(safePath.parent)(ServerFileSystemContext.project, workspace)
extractArchiveFromFiles(archiveFile, toFile)(ServerFileSystemContext.project)
......@@ -336,7 +339,13 @@ class ApiImpl(s: Services, applicationControl: ApplicationControl) extends Api {
t match {
case ce: ScalaREPL.CompilationError ⇒
def toErrorWithLocation(em: ScalaREPL.ErrorMessage) =
ErrorWithLocation(em.rawMessage, em.position.map { _.line }, em.position.map { _.start }, em.position.map { _.end })
ErrorWithLocation(em.rawMessage, em.position.map {
_.line
}, em.position.map {
_.start
}, em.position.map {
_.end
})
ErrorData(ce.errorMessages.map(toErrorWithLocation), t)
case _ ⇒ ErrorData(t)
......@@ -353,14 +362,18 @@ class ApiImpl(s: Services, applicationControl: ApplicationControl) extends Api {
case ErrorInCode(e) ⇒ Some(error(e))
case ErrorInCompiler(e) ⇒ Some(error(e))
case compiled: Compiled ⇒
onCompiled.foreach { _(execId) }
onCompiled.foreach {
_(execId)
}
catchAll(OutputManager.withStreamOutputs(outputStream, outputStream)(compiled.eval)) match {
case Failure(e) ⇒ Some(error(e))
case Success(dsl) ⇒
val services = MoleServices.copy(MoleServices.create)(outputRedirection = OutputRedirection(outputStream))
Try(dslToPuzzle(dsl).toExecution()(services)) match {
case Success(ex) ⇒
onEvaluated.foreach { _(ex, execId) }
onEvaluated.foreach {
_(ex, execId)
}
None
case Failure(e) ⇒ Some(error(e))
}
......@@ -536,6 +549,7 @@ class ApiImpl(s: Services, applicationControl: ApplicationControl) extends Api {
gridscale.http.getResponse(checkedURL) { response ⇒
def extractName = checkedURL.split("/").last
val name =
response.headers.flatMap {
case ("Content-Disposition", value) ⇒
......
openmole/project/Libraries.scala
View file @ 1dc4937f
......@@ -103,6 +103,7 @@ object Libraries {
lazy val sourceCode = "org.openmole.library" %% "sourcecode" % sourcecodeVersion
lazy val txtmark = "org.openmole.library" %% "com-github-rjeschke-txtmark" % "0.13"
lazy val spatialdata = "org.openmole.library" %% "org-openmole-spatialdata" % spatialdataVersion
lazy val xzJava = "org.openmole.library" %% "xzjava" % "1.8"
def httpClientVersion = "4.5.3"
lazy val httpClient =
......
openmole/third-parties/org.openmole.tool.tar/src/main/scala/org/openmole/tool/tar/package.scala
View file @ 1dc4937f
......@@ -16,11 +16,14 @@
*/
package org.openmole.tool
import java.io.{ IOException, File }
import java.io._
import java.nio.file._
import org.openmole.tool.file._
import org.openmole.tool.stream._
import scala.collection.mutable.{ Stack, ListBuffer }
import org.tukaani.xz.{ LZMA2Options, XZInputStream, XZOutputStream }
import scala.collection.mutable.{ ListBuffer, Stack }
import scala.collection.JavaConverters._
package object tar {
......@@ -87,6 +90,48 @@ package object tar {
}
}
implicit class XZFileDecorator(file: File) {
def extractXZ(to: File) = {
if (!file.getName.endsWith(".xz")) throw new java.io.IOException(s"$file is not a XZ file")
else {
val inputStream = new FileInputStream(file)
val outputStream = new FileOutputStream(to)
val inxz = extractToStream
val buffer = new Array[Byte](inputStream.available)
Iterator.continually(inxz.read(buffer)).takeWhile(_ != -1).foreach {
outputStream.write(buffer, 0, _)
}
inxz.close
}
}
def extractToStream: InputStream = {
if (!file.getName.endsWith(".xz")) throw new java.io.IOException(s"$file is not a XZ file")
else {
val inputStream = new FileInputStream(file)
new XZInputStream(inputStream, 100 * 1024)
}
}
def compressXZ(to: File) = {
val outfile = new FileOutputStream(to)
val outxz = new XZOutputStream(outfile, new LZMA2Options(8), org.tukaani.xz.XZ.CHECK_SHA256)
val infile = new FileInputStream(file)
val buffer = new Array[Byte](8192)
Iterator.continually(infile.read(buffer)).takeWhile(_ != -1).foreach { size ⇒
outxz.write(buffer, 0, size)
}
outxz.finish
}
}
implicit class FileTarArchiveDecorator(file: File) {
def archive(dest: File, time: Boolean = true) =
......@@ -110,6 +155,12 @@ package object tar {
_.extract(dest, overwrite)
}
def extractUncompressXZ(dest: File, overwrite: Boolean = false) = {
withClosable(new TarInputStream(file.extractToStream)) {
_.extract(dest, overwrite)
}
}
def copyCompress(toF: File): File = {
if (toF.isDirectory) file.archiveCompress(toF)
else file.copyCompressFile(toF)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment