Skip to content

GitLab

Commit c4694ed4 authored by Romain Reuillon's avatar Romain Reuillon
Browse files

[Core] fix: clousure delegation to remote env with scala 2.12.

parent 279affd5
Hide whitespace changes
Inline Side-by-side
openmole/build.sbt
View file @ c4694ed4
......@@ -253,14 +253,13 @@ lazy val outputManager = OsgiProject(coreDir, "org.openmole.core.outputmanager",
lazy val outputRedirection = OsgiProject(coreDir, "org.openmole.core.outputredirection", imports = Seq("*")) settings (coreSettings: _*)
lazy val console = OsgiProject(coreDir, "org.openmole.core.console", global = true, imports = Seq("*")) dependsOn
(pluginManager) settings(
lazy val console = OsgiProject(coreDir, "org.openmole.core.console", global = true, imports = Seq("*")) dependsOn (pluginManager) settings(
OsgiKeys.importPackage := Seq("*"),
Libraries.addScalaLang(scalaVersionValue),
libraryDependencies += Libraries.monocle,
macroParadise,
defaultActivator
) dependsOn(openmoleByteCode, openmoleOSGi, workspace, fileService) settings (coreSettings: _*)
) dependsOn(openmoleOSGi, workspace, fileService) settings (coreSettings: _*)
lazy val project = OsgiProject(coreDir, "org.openmole.core.project", imports = Seq("*")) dependsOn(console, openmoleDSL, services) settings (OsgiKeys.importPackage := Seq("*")) settings (coreSettings: _*)
......@@ -383,7 +382,8 @@ def allEnvironment = Seq(batch, gridscale, ssh, egi, pbs, oar, sge, condor, slur
lazy val batch = OsgiProject(pluginDir, "org.openmole.plugin.environment.batch", imports = Seq("*")) dependsOn(
workflow, workspace, tools, event, replication, exception,
serializer, fileService, pluginManager, openmoleTar, communication, authentication, location, services
serializer, fileService, pluginManager, openmoleTar, communication, authentication, location, services,
openmoleByteCode
) settings (
libraryDependencies ++= Seq(
Libraries.gridscale,
......@@ -714,7 +714,7 @@ def binDir = file("bin")
def bundleFilter(m: ModuleID, artifact: Artifact) = {
def excludedLibraryDependencies = Set("slick", "squants", "shapeless")
def excludedLibraryDependencies = Set("slick", "squants", "shapeless", "sourcecode")
def exclude =
(m.organization != "org.openmole.library" && excludedLibraryDependencies.exists(m.name.contains)) ||
......
openmole/core/org.openmole.core.console/src/main/scala/org/openmole/core/console/ScalaREPL.scala
View file @ c4694ed4
......@@ -73,7 +73,7 @@ object ScalaREPL {
case class BundledClass(name: String, bundle: Option[Bundle])
case class REPLClass(name: String, path: String, classLoader: REPLClassloader) {
def byteCode = classLoader.findClassFile(name).getOrElse(throw new InternalProcessingError("Class not found in this classloader")).toByteArray
// def byteCode = classLoader.findClassFile(name).getOrElse(throw new InternalProcessingError("Class not found in this classloader")).toByteArray
}
case class ReferencedClasses(repl: Vector[REPLClass], other: Vector[BundledClass]) {
def plugins = other.flatMap(_.bundle).flatMap(PluginManager.allPluginDependencies).distinct.map(_.file)
......@@ -85,28 +85,28 @@ object ScalaREPL {
def empty = ReferencedClasses(Vector.empty, Vector.empty)
}
def bundleFromReferencedClass(ref: ReferencedClasses, bundleName: String, bundleVersion: String, bundle: java.io.File) = {
val classByteCode = ref.repl.distinct.map(c ⇒ ClassByteCode(c.path, c.byteCode))
def packageName(c: String) = c.reverse.dropWhile(_ != '.').drop(1).reverse
def importPackages = ref.other.filter(_.bundle.isDefined).groupBy(_.bundle).toSeq.flatMap {
case (b, cs) ⇒
for {
c ← cs
p = packageName(c.name)
if !p.isEmpty
} yield VersionedPackage(p, b.map(_.getVersion.toString))
}.distinct
createBundle(
name = bundleName,
version = bundleVersion,
classes = classByteCode,
exportedPackages = ref.repl.map(c ⇒ packageName(c.name)).distinct.filter(p ⇒ !p.isEmpty),
importedPackages = importPackages,
bundle = bundle
)
}
// def bundleFromReferencedClass(ref: ReferencedClasses, bundleName: String, bundleVersion: String, bundle: java.io.File) = {
// val classByteCode = ref.repl.distinct.map(c ⇒ ClassByteCode(c.path, c.byteCode))
//
// def packageName(c: String) = c.reverse.dropWhile(_ != '.').drop(1).reverse
// def importPackages = ref.other.filter(_.bundle.isDefined).groupBy(_.bundle).toSeq.flatMap {
// case (b, cs) ⇒
// for {
// c ← cs
// p = packageName(c.name)
// if !p.isEmpty
// } yield VersionedPackage(p, b.map(_.getVersion.toString))
// }.distinct
//
// createBundle(
// name = bundleName,
// version = bundleVersion,
// classes = classByteCode,
// exportedPackages = ref.repl.map(c ⇒ packageName(c.name)).distinct.filter(p ⇒ !p.isEmpty),
// importedPackages = importPackages,
// bundle = bundle
// )
// }
class OMIMain(settings: Settings, priorityBundles: ⇒ Seq[Bundle], jars: Seq[JFile], quiet: Boolean) extends IMain(settings) {
var storeErrors: Boolean = true
......@@ -185,58 +185,58 @@ import ScalaREPL._
class REPLClassloader(val file: AbstractFile, parent: ClassLoader) extends scala.reflect.internal.util.AbstractFileClassLoader(file, parent) { cl ⇒
def classFiles = {
def all(fs: AbstractFile): List[AbstractFile] = {
if (fs.isDirectory) fs :: fs.iterator.flatMap(all).toList
else List(fs)
}
all(file).filterNot(_.isDirectory)
}
def toClassPath(c: String) = s"${c.replace('.', '/')}.class"
def toAbsoluteClassPath(c: String) = s"${file.name}/${toClassPath(c)}"
def findClassFile(name: String): Option[AbstractFile] = classFiles.find(_.path == toAbsoluteClassPath(name))
def findClassFile(c: Class[_]): Option[AbstractFile] = findClassFile(c.getClass.getName)
def referencedClasses(
classes: Seq[Class[_]],
additionalClasses: Seq[Class[_]] = Seq(classOf[scala.runtime.AbstractFunction1[_, _]])
) = {
import org.openmole.tool.bytecode._
val seen = collection.mutable.HashSet[String]()
val toProcess = collection.mutable.Stack[String]()
classes.foreach(c ⇒ seen.add(c.getName))
classes.foreach(c ⇒ toProcess.push(c.getName))
additionalClasses.foreach(c ⇒ seen.add(c.getName))
while (!toProcess.isEmpty) {
val processing = toProcess.pop
for {
classFile ← findClassFile(processing).toSeq
refClass ← listAllClasses(classFile.toByteArray)
if !seen.contains(refClass.getClassName)
} {
seen += refClass.getClassName
toProcess push refClass.getClassName
}
}
val (repl, other) = seen.toVector.sorted.partition(c ⇒ findClassFile(c).isDefined)
val replClasses = repl.map { c ⇒
REPLClass(c, cl.toClassPath(c), cl)
}
val bundledOther = other.map { c ⇒
val bundle = tryToLoadClass(c).flatMap(PluginManager.bundleForClass)
BundledClass(name = c, bundle = bundle)
}
// def classFiles = {
// def all(fs: AbstractFile): List[AbstractFile] = {
// if (fs.isDirectory) fs :: fs.iterator.flatMap(all).toList
// else List(fs)
// }
// all(file).filterNot(_.isDirectory)
// }
ReferencedClasses(replClasses, bundledOther)
}
// def toClassPath(c: String) = s"${c.replace('.', '/')}.class"
// def toAbsoluteClassPath(c: String) = s"${file.name}/${toClassPath(c)}"
// def findClassFile(name: String): Option[AbstractFile] = classFiles.find(_.path == toAbsoluteClassPath(name))
// def findClassFile(c: Class[_]): Option[AbstractFile] = findClassFile(c.getClass.getName)
//
// def referencedClasses(
// classes: Seq[Class[_]],
// additionalClasses: Seq[Class[_]] = Seq(classOf[scala.runtime.AbstractFunction1[_, _]])
// ) = {
//
// import org.openmole.tool.bytecode._
//
// val seen = collection.mutable.HashSet[String]()
// val toProcess = collection.mutable.Stack[String]()
//
// classes.foreach(c ⇒ seen.add(c.getName))
// classes.foreach(c ⇒ toProcess.push(c.getName))
// additionalClasses.foreach(c ⇒ seen.add(c.getName))
//
// while (!toProcess.isEmpty) {
// val processing = toProcess.pop
// for {
// classFile ← findClassFile(processing).toSeq
// refClass ← listAllClasses(classFile.toByteArray)
// if !seen.contains(refClass.getClassName)
// } {
// seen += refClass.getClassName
// toProcess push refClass.getClassName
// }
// }
//
// val (repl, other) = seen.toVector.sorted.partition(c ⇒ findClassFile(c).isDefined)
//
// val replClasses = repl.map { c ⇒
// REPLClass(c, cl.toClassPath(c), cl)
// }
//
// val bundledOther = other.map { c ⇒
// val bundle = tryToLoadClass(c).flatMap(PluginManager.bundleForClass)
// BundledClass(name = c, bundle = bundle)
// }
//
// ReferencedClasses(replClasses, bundledOther)
// }
}
......
openmole/core/org.openmole.core.project/src/main/scala/org/openmole/core/project/Project.scala
View file @ c4694ed4
......@@ -113,8 +113,9 @@ case class ErrorInCode(error: ScalaREPL.CompilationError) extends CompilationErr
case class ErrorInCompiler(error: Throwable) extends CompilationError
case class Compiled(result: ScalaREPL.Compiled) extends CompileResult {
def eval =
result.apply() match {
result.apply().asInstanceOf[Function0[_]]() match {
case p: Puzzle ⇒ p
case e ⇒ throw new UserBadDataError(s"Script should end with a workflow (it ends with ${if (e == null) null else e.getClass}).")
}
......@@ -139,7 +140,7 @@ class Project(workDirectory: File, newREPL: (ConsoleVariables) ⇒ ScalaREPL) {
def footer =
s"""}
|runOMSScript()""".stripMargin
|() => runOMSScript()""".stripMargin
def compileContent =
s"""$header
......
openmole/core/org.openmole.core.serializer/src/main/scala/org/openmole/core/serializer/PluginAndFilesListing.scala
View file @ c4694ed4
......@@ -21,15 +21,15 @@ import java.io.File
import com.thoughtworks.xstream.converters.Converter
import com.thoughtworks.xstream.converters.reflection.ReflectionConverter
import com.thoughtworks.xstream.core.{ JVM, ClassLoaderReference }
import com.thoughtworks.xstream.core.{ ClassLoaderReference, JVM }
import com.thoughtworks.xstream.core.util.CompositeClassLoader
import com.thoughtworks.xstream.io.xml.XppDriver
import com.thoughtworks.xstream.{ XStream, mapper }
import com.thoughtworks.xstream.mapper.{ DefaultMapper, MapperWrapper, Mapper }
import com.thoughtworks.xstream.mapper.{ DefaultMapper, Mapper, MapperWrapper }
import org.openmole.core.pluginmanager.PluginManager
import org.openmole.core.serializer.converter.Serialiser
import org.openmole.core.serializer.file.FileConverterNotifier
import org.openmole.core.serializer.plugin.{ Plugins, PluginClassConverter, PluginConverter }
import org.openmole.core.serializer.plugin.{ PluginClassConverter, PluginConverter, Plugins }
import org.openmole.core.serializer.structure.PluginClassAndFiles
import org.openmole.tool.file._
import org.openmole.tool.stream.NullOutputStream
......@@ -37,6 +37,7 @@ import org.openmole.core.console._
import scala.collection.immutable.{ HashSet, TreeSet }
import scala.collection.mutable
import scala.reflect.internal.util.ScalaClassLoader.URLClassLoader
trait PluginAndFilesListing { this: Serialiser ⇒
......@@ -58,7 +59,7 @@ trait PluginAndFilesListing { this: Serialiser ⇒
PluginManager.pluginsForClass(c).foreach(pluginUsed)
}
if (c.getClassLoader != null && classOf[REPLClassloader].isAssignableFrom(c.getClassLoader.getClass)) {
if (c.getClassLoader != null && classOf[URLClassLoader].isAssignableFrom(c.getClassLoader.getClass) && !PluginManager.bundleForClass(c).isDefined) {
replClasses += c
}
......
openmole/plugins/org.openmole.plugin.environment.batch/src/main/scala/org/openmole/plugin/environment/batch/environment/BatchEnvironment.scala
View file @ c4694ed4
......@@ -28,6 +28,7 @@ import org.openmole.core.event.{Event, EventDispatcher}
import org.openmole.core.exception.UserBadDataError
import org.openmole.core.fileservice.{FileCache, FileService, FileServiceCache}
import org.openmole.core.location._
import org.openmole.core.outputmanager.OutputManager
import org.openmole.core.pluginmanager.PluginManager
import org.openmole.core.preference.{ConfigurationLocation, Preference}
import org.openmole.core.replication.ReplicaCatalog
......@@ -37,6 +38,7 @@ import org.openmole.core.workflow.execution._
import org.openmole.core.workflow.job._
import org.openmole.core.workspace._
import org.openmole.plugin.environment.batch.environment.BatchEnvironment.ExecutionJobRegistry
import org.openmole.plugin.environment.batch.environment.BatchExecutionJob.ClosuresBundle
import org.openmole.plugin.environment.batch.refresh._
import org.openmole.tool.cache._
import org.openmole.tool.file._
......@@ -47,6 +49,9 @@ import squants.information.InformationConversions._
import squants.time.TimeConversions._
import scala.collection.immutable.TreeSet
import scala.reflect.internal.util.ScalaClassLoader.URLClassLoader
import org.openmole.tool.osgi._
import org.osgi.framework.Bundle
object BatchEnvironment extends JavaLogger {
......@@ -191,7 +196,7 @@ object BatchEnvironment extends JavaLogger {
serializerService.serialise(job.runnableTasks, jobFile)
val plugins = new TreeSet[File]()(fileOrdering) ++ job.plugins
val plugins = new TreeSet[File]()(fileOrdering) ++ job.plugins -- job.environment.plugins
val files = (new TreeSet[File]()(fileOrdering) ++ job.files) diff plugins
val runtime = replicateTheRuntime(job.job, job.environment, replicate)
......@@ -331,7 +336,7 @@ abstract class BatchEnvironment extends SubmissionEnvironment { env ⇒
lazy val registry = new ExecutionJobRegistry()
def jobs = ExecutionJobRegistry.executionJobs(registry)
lazy val replBundleCache = new AssociativeCache[ReferencedClasses, FileCache]()
lazy val replBundleCache = new AssociativeCache[ClosuresBundle, FileCache]()
lazy val plugins = PluginManager.pluginsForClass(this.getClass)
......@@ -358,35 +363,114 @@ abstract class BatchEnvironment extends SubmissionEnvironment { env ⇒
object BatchExecutionJob {
def apply(job: Job, environment: BatchEnvironment) = new BatchExecutionJob(job, environment)
def toClassPath(c: String) = s"${c.replace('.', '/')}.class"
def toClassName(p: String) = p.dropRight(".class".size).replace("/", ".")
def replClassDirectory(c: Class[_]) = {
val replClassloader = c.getClassLoader.asInstanceOf[URLClassLoader]
val location = toClassPath(c.getName)
val classURL = replClassloader.findResource(location)
new File(classURL.getPath.dropRight(location.size))
}
def allClasses(directory: File): Seq[ClassFile] = {
import java.nio.file._
import collection.JavaConverters._
Files.walk(directory.toPath).
filter(p => Files.isRegularFile(p) && p.toFile.getName.endsWith(".class")).iterator().asScala.
map { p =>
val path = directory.toPath.relativize(p)
ClassFile(path.toString, p.toFile)
}.toList
}
def isREPLPackage(p: String) = p.startsWith("$line")
case class ClosuresBundle(classes: Seq[ClassFile], exported: Seq[String], dependencies: Seq[VersionedPackage], plugins: Seq[File])
}
class BatchExecutionJob(val job: Job, val environment: BatchEnvironment) extends ExecutionJob { bej ⇒
def moleJobs = job.moleJobs
def runnableTasks = job.moleJobs.map(RunnableTask(_))
def plugins = pluginsAndFiles.plugins ++ closureBundle.map(_.file) ++ referencedClosures.toSeq.flatMap(_.plugins)
def plugins = pluginsAndFiles.plugins ++ closureBundle.map(_.file) ++ replClosuresBundles.flatMap(_.plugins)
def files = pluginsAndFiles.files
@transient lazy val pluginsAndFiles = environment.services.serializerService.pluginsAndFiles(runnableTasks)
@transient lazy val referencedClosures = {
if (pluginsAndFiles.replClasses.isEmpty) None
else {
def referenced =
pluginsAndFiles.replClasses.map { c ⇒
val replClassloader = c.getClassLoader.asInstanceOf[REPLClassloader]
replClassloader.referencedClasses(Seq(c))
}.fold(ReferencedClasses.empty)(ReferencedClasses.merge)
Some(referenced)
@transient lazy val replClosuresBundles = {
import java.nio.file._
import org.openmole.tool.bytecode._
val replDirectories =
pluginsAndFiles.replClasses.map(c => c.getClassLoader -> BatchExecutionJob.replClassDirectory(c)).distinct
def bundles(directory: File, classLoader: ClassLoader) = {
val allClassFiles = BatchExecutionJob.allClasses(directory)
val mentionedClasses =
for {
f <- allClassFiles.toList
t <- listAllClasses(Files.readAllBytes(f.file))
c <- util.Try[Class[_]](classLoader.loadClass(t.getClassName)).toOption.toSeq
} yield BatchExecutionJob.toClassName(f.path) -> c
def toVersionedPackage(c: Class[_]) = {
val p = c.getPackage.getName
PluginManager.bundleForClass(c).map { b => VersionedPackage(p, Some(b.getVersion.toString)) }
}
val packages = mentionedClasses.map(_._2).flatMap(toVersionedPackage).distinct
val plugins = mentionedClasses.map(_._2).flatMap(PluginManager.pluginsForClass)
val exported =
allClassFiles.flatMap(c => Option(new File(c.path).getParent)).distinct.
filter(BatchExecutionJob.isREPLPackage).
map(_.replace("/", "."))
// def replClassFiles() = {
// val replClasses = pluginsAndFiles.replClasses.map(c => c.getName).toSet
//
// val mentions = Map[String, Class[_]]() ++ mentionedClasses
// val primaryREPLClasses = replClasses.filter(mentions.isDefinedAt)
//
// val seen = collection.mutable.HashSet[String]()
// val toProcess = collection.mutable.Stack[String]()
//
// primaryREPLClasses.foreach(c ⇒ seen.add(c))
// primaryREPLClasses.foreach(c ⇒ toProcess.push(c))
//
// while (!toProcess.isEmpty) {
// val processing = toProcess.pop
// for {
// clazz ← mentions.get(processing).toSeq
// if !seen.contains(clazz.getName)
// } {
// seen += clazz.getName
// toProcess push clazz.getName
// }
// }
//
// allClassFiles.filter(c => seen.contains(BatchExecutionJob.toClassName(c.path)))
// }
BatchExecutionJob.ClosuresBundle(allClassFiles, exported, packages, plugins)
}
}
def closureBundle =
referencedClosures.map { closures ⇒
replDirectories.map { case(c, d) => bundles(d, c) }
}
def closureBundle: Seq[FileCache] =
replClosuresBundles.map { closures ⇒
environment.replBundleCache.cache(job.moleExecution, closures, preCompute = false) { rc ⇒
val bundle = environment.services.newFile.newFile("closureBundle", ".jar")
try ScalaREPL.bundleFromReferencedClass(closures, "closure-" + UUID.randomUUID.toString, "1.0", bundle)
try createBundle("closure-" + UUID.randomUUID.toString, "1.0", closures.classes, closures.exported, closures.dependencies, bundle)
catch {
case e: Throwable ⇒
bundle.delete()
......
openmole/project/Libraries.scala
View file @ c4694ed4
......@@ -18,7 +18,7 @@ object Libraries {
lazy val rxVersion = "0.4.0"
lazy val boopickleVersion = "1.2.6"
lazy val scalaAutowireVersion = "0.2.6"
lazy val sourcecodeVersion = "0.1.3"
lazy val sourcecodeVersion = "0.1.4"
lazy val scaladgetVersion = "1.1.0"
lazy val sortableVersion = "0.2.1"
lazy val json4sVersion = "3.5.0"
......
openmole/third-parties/org.openmole.tool.bytecode/src/main/scala/org/openmole/tool/bytecode/package.scala
View file @ c4694ed4
......@@ -22,7 +22,7 @@ import scala.collection._
package object bytecode {
def listAllClasses(byteCode: Array[Byte]) = {
def listAllClasses(byteCode: Array[Byte]): List[Type] = {
val classes = mutable.HashSet[Type]()
val cv = new ClassVisitor(Opcodes.ASM5) {
......
openmole/third-parties/org.openmole.tool.osgi/src/main/scala/org/openmole/tool/osgi/package.scala
View file @ c4694ed4
......@@ -23,17 +23,34 @@ import java.util.zip.ZipEntry
import org.openmole.tool.file._
import org.openmole.tool.stream._
import org.osgi.framework.Constants
import org.osgi.framework.{ Bundle, Constants }
package object osgi {
case class ClassByteCode(name: String, byteCode: Array[Byte])
object ClassSource {
def path(c: ClassSource) =
c match {
case ClassFile(path, _) ⇒ path
case ClassByteCode(path, _) ⇒ path
}
def openInputStream(c: ClassSource) =
c match {
case ClassFile(_, file) ⇒ file.bufferedInputStream
case ClassByteCode(_, byteCode) ⇒ new ByteArrayInputStream(byteCode)
}
}
sealed class ClassSource
case class ClassFile(path: String, file: File) extends ClassSource
case class ClassByteCode(path: String, byteCode: Array[Byte]) extends ClassSource
case class VersionedPackage(name: String, version: Option[String])
def createBundle(
name: String,
version: String,
classes: Seq[ClassByteCode],
classes: Seq[ClassSource],
exportedPackages: Seq[String],
importedPackages: Seq[VersionedPackage],
bundle: File
......@@ -56,8 +73,9 @@ package object osgi {
for {
cbc ← classes
} {
os.putNextEntry(new ZipEntry(cbc.name))
copy(new ByteArrayInputStream(cbc.byteCode), os)
os.putNextEntry(new ZipEntry(ClassSource.path(cbc)))
val is = ClassSource.openInputStream(cbc)
try copy(is, os) finally is.close()
os.closeEntry()
}
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment