Commit 3fd898ed authored by Romain Reuillon's avatar Romain Reuillon
Browse files

Integrate gridscale 2 and refactoring environment. First step, ssh environment is working.

parent 0edfd1aa
......@@ -327,3 +327,31 @@ lazy val sourceCode = OsgiProject(dir, "sourcecode") settings (
libraryDependencies += "com.lihaoyi" %% "sourcecode" % "0.1.3",
version := "0.1.3"
) settings(settings: _*)
def gridscaleVersion = "2.0-SNAPSHOT"
lazy val gridscale = OsgiProject(dir, "gridscale", imports = Seq("*")) settings (
libraryDependencies += "fr.iscpif.gridscale" %% "gridscale" % gridscaleVersion,
version := gridscaleVersion
) settings(settings: _*) dependsOn(freedsl)
lazy val gridscaleLocal = OsgiProject(dir, "gridscale.local", imports = Seq("*")) settings (
libraryDependencies += "fr.iscpif.gridscale" %% "local" % gridscaleVersion,
version := gridscaleVersion
) settings(settings: _*) dependsOn(gridscale)
lazy val gridscaleHTTP = OsgiProject(dir, "gridscale.http") settings (
libraryDependencies += "fr.iscpif.gridscale" %% "http" % gridscaleVersion,
version := gridscaleVersion
) settings(settings: _*) dependsOn(gridscale)
lazy val gridscaleSSH = OsgiProject(dir, "gridscale.ssh", imports = Seq("*"), privatePackages = Seq("gridscale.http.*"/*, "net.schmizz.*", "com.hierynomus.sshj.*", "com.jcraft.jzlib.*", "djb.*", "net.i2p.crypto.*", "sshj.properties"*/)) settings (
libraryDependencies += "fr.iscpif.gridscale" %% "ssh" % gridscaleVersion,
version := gridscaleVersion
) settings(settings: _*) dependsOn(jzlib) dependsOn(gridscale)
lazy val jzlib = OsgiProject(dir, "com.jcraft.jzlib", imports = Seq("*")) settings (
libraryDependencies += "com.jcraft" % "jzlib" % "1.1.3",
version := "1.1.3"
) settings(settings: _*)
\ No newline at end of file
......@@ -46,7 +46,9 @@ object SimExplorer extends Logger {
path: Option[String] = None,
pluginPath: Option[String] = None,
nbThread: Option[Int] = None,
workspace: Option[String] = None
workspace: Option[String] = None,
test: Boolean = false,
debug: Boolean = false
)
val parser = new OptionParser[Config]("OpenMOLE") {
......@@ -72,41 +74,49 @@ object SimExplorer extends Logger {
opt[String]('w', "workspace") text ("Workspace location") action {
(v, c) c.copy(workspace = Some(v))
}
opt[Unit]('d', "debug") text ("Switch on the debug mode") action {
(_, c) c.copy(debug = true)
}
opt[Unit]("test") text ("Switch on test mode") action {
(_, c) c.copy(test = true)
}
}
val debug = args.contains("-d")
val filteredArgs = args.filterNot((_: String) == "-d")
if (debug) LoggerService.level("ALL")
parser.parse(filteredArgs, Config()) foreach { config
implicit val workspace = Workspace(new File(config.workspace.get).getCanonicalFile)
implicit val newFile = NewFile(workspace)
implicit val serializerService = SerializerService()
implicit val preference = Preference.memory()
implicit val threadProvider = ThreadProvider()
implicit val fileService = FileService()
implicit val eventDispatcher = EventDispatcher()
try {
PluginManager.startAll.foreach { case (b, e) logger.log(WARNING, s"Error starting bundle $b", e) }
logger.fine("plugins: " + config.pluginPath.get + " " + new File(config.pluginPath.get).listFilesSafe.mkString(","))
PluginManager.tryLoad(new File(config.pluginPath.get).listFilesSafe).foreach { case (f, e) logger.log(WARNING, s"Error loading bundle $f", e) }
val (storage, _) = serializerService.deserialiseAndExtractFiles[RemoteStorage](new File(config.storage.get))
new Runtime().apply(
storage,
config.path.get,
config.inputMessage.get,
config.outputMessage.get,
config.nbThread.getOrElse(1),
debug
)
}
finally {
threadProvider.stop()
parser.parse(args, Config()) foreach { config
config.test match {
case false
if (config.debug) LoggerService.level("ALL")
implicit val workspace = Workspace(new File(config.workspace.get).getCanonicalFile)
implicit val newFile = NewFile(workspace)
implicit val serializerService = SerializerService()
implicit val preference = Preference.memory()
implicit val threadProvider = ThreadProvider()
implicit val fileService = FileService()
implicit val eventDispatcher = EventDispatcher()
try {
PluginManager.startAll.foreach { case (b, e) logger.log(WARNING, s"Error starting bundle $b", e) }
logger.fine("plugins: " + config.pluginPath.get + " " + new File(config.pluginPath.get).listFilesSafe.mkString(","))
PluginManager.tryLoad(new File(config.pluginPath.get).listFilesSafe).foreach { case (f, e) logger.log(WARNING, s"Error loading bundle $f", e) }
val (storage, _) = serializerService.deserialiseAndExtractFiles[RemoteStorage](new File(config.storage.get))
new Runtime().apply(
storage,
config.path.get,
config.inputMessage.get,
config.outputMessage.get,
config.nbThread.getOrElse(1),
config.debug
)
}
finally {
threadProvider.stop()
}
case true logger.info("The runtime is working")
}
}
......
......@@ -201,12 +201,12 @@ lazy val fileService = OsgiProject(coreDir, "org.openmole.core.fileservice", imp
lazy val threadProvider = OsgiProject(coreDir, "org.openmole.core.threadprovider", imports = Seq("*")) dependsOn(tools, preference) settings (coreSettings: _*) settings (defaultActivator)
lazy val module = OsgiProject(coreDir, "org.openmole.core.module", imports = Seq("*")) dependsOn(buildinfo, expansion, openmoleHash, openmoleFile, pluginManager) settings (coreSettings: _*) settings(
libraryDependencies += Libraries.gridscaleHTTP,
libraryDependencies ++= Libraries.gridscaleHTTP,
libraryDependencies += Libraries.json4s,
defaultActivator)
lazy val market = OsgiProject(coreDir, "org.openmole.core.market", imports = Seq("*")) enablePlugins (ScalaJSPlugin) dependsOn(buildinfo, expansion, openmoleHash, openmoleFile, pluginManager) settings (coreSettings: _*) settings(
libraryDependencies += Libraries.gridscaleHTTP,
libraryDependencies ++= Libraries.gridscaleHTTP,
libraryDependencies += Libraries.json4s,
defaultActivator)
......@@ -329,7 +329,7 @@ lazy val rangeDomain = OsgiProject(pluginDir, "org.openmole.plugin.domain.range"
/* Environment */
def allEnvironment = Seq(batch, oar, desktopgrid, egi, gridscale, pbs, sge, condor, slurm, ssh)
def allEnvironment = Seq(batch, gridscale, ssh) //, oar, desktopgrid, egi, gridscale, pbs, sge, condor, slurm, ssh)
lazy val batch = OsgiProject(pluginDir, "org.openmole.plugin.environment.batch", imports = Seq("*")) dependsOn(
workflow, workspace, tools, event, replication, exception,
......@@ -344,35 +344,37 @@ lazy val batch = OsgiProject(pluginDir, "org.openmole.plugin.environment.batch",
)
) settings (pluginSettings: _*)
lazy val oar = OsgiProject(pluginDir, "org.openmole.plugin.environment.oar", imports = Seq("*")) dependsOn(openmoleDSL, batch, gridscale, ssh) settings
(libraryDependencies += Libraries.gridscaleOAR) settings (pluginSettings: _*)
lazy val desktopgrid = OsgiProject(pluginDir, "org.openmole.plugin.environment.desktopgrid", imports = Seq("*")) dependsOn(
openmoleDSL,
batch, sftpserver, gridscale
) settings (pluginSettings: _*)
lazy val egi = OsgiProject(pluginDir, "org.openmole.plugin.environment.egi", imports = Seq("!org.apache.http.*", "!fr.iscpif.gridscale.libraries.srmstub", "!fr.iscpif.gridscale.libraries.lbstub", "!fr.iscpif.gridscale.libraries.wmsstub", "!com.google.common.cache", "*")) dependsOn(openmoleDSL,
batch,
workspace, fileService, gridscale) settings (
libraryDependencies ++= Seq(Libraries.gridscaleGlite, Libraries.gridscaleHTTP),
Libraries.addScalaLang
) settings (pluginSettings: _*)
lazy val gridscale = OsgiProject(pluginDir, "org.openmole.plugin.environment.gridscale", imports = Seq("*")) dependsOn(openmoleDSL, tools,
batch, exception) settings (pluginSettings: _*)
lazy val pbs = OsgiProject(pluginDir, "org.openmole.plugin.environment.pbs", imports = Seq("*")) dependsOn(openmoleDSL, batch, gridscale, ssh) settings
(libraryDependencies += Libraries.gridscalePBS) settings (pluginSettings: _*)
lazy val sge = OsgiProject(pluginDir, "org.openmole.plugin.environment.sge", imports = Seq("*")) dependsOn(openmoleDSL, batch, gridscale, ssh) settings
(libraryDependencies += Libraries.gridscaleSGE) settings (pluginSettings: _*)
lazy val condor = OsgiProject(pluginDir, "org.openmole.plugin.environment.condor", imports = Seq("*")) dependsOn(openmoleDSL, batch, gridscale, ssh) settings
(libraryDependencies += Libraries.gridscaleCondor) settings (pluginSettings: _*)
lazy val slurm = OsgiProject(pluginDir, "org.openmole.plugin.environment.slurm", imports = Seq("*")) dependsOn(openmoleDSL, batch, gridscale, ssh) settings
(libraryDependencies += Libraries.gridscaleSLURM) settings (pluginSettings: _*)
//lazy val oar = OsgiProject(pluginDir, "org.openmole.plugin.environment.oar", imports = Seq("*")) dependsOn(openmoleDSL, batch, gridscale, ssh) settings
// (libraryDependencies += Libraries.gridscaleOAR) settings (pluginSettings: _*)
//
//lazy val desktopgrid = OsgiProject(pluginDir, "org.openmole.plugin.environment.desktopgrid", imports = Seq("*")) dependsOn(
// openmoleDSL,
// batch, sftpserver, gridscale
//) settings (pluginSettings: _*)
//
//lazy val egi = OsgiProject(pluginDir, "org.openmole.plugin.environment.egi", imports = Seq("!org.apache.http.*", "!fr.iscpif.gridscale.libraries.srmstub", "!fr.iscpif.gridscale.libraries.lbstub", "!fr.iscpif.gridscale.libraries.wmsstub", "!com.google.common.cache", "*")) dependsOn(openmoleDSL,
// batch,
// workspace, fileService, gridscale) settings (
// libraryDependencies ++= Seq(Libraries.gridscaleGlite, Libraries.gridscaleHTTP),
// Libraries.addScalaLang,
// ) settings (pluginSettings: _*)
//
lazy val gridscale = OsgiProject(pluginDir, "org.openmole.plugin.environment.gridscale", imports = Seq("*")) settings(
libraryDependencies += Libraries.gridscaleLocal)dependsOn(openmoleDSL, tools, batch, exception) settings (pluginSettings: _*)
//lazy val pbs = OsgiProject(pluginDir, "org.openmole.plugin.environment.pbs", imports = Seq("*")) dependsOn(openmoleDSL, batch, gridscale, ssh) settings
// (libraryDependencies += Libraries.gridscalePBS) settings (pluginSettings: _*)
//
//lazy val sge = OsgiProject(pluginDir, "org.openmole.plugin.environment.sge", imports = Seq("*")) dependsOn(openmoleDSL, batch, gridscale, ssh) settings
// (libraryDependencies += Libraries.gridscaleSGE) settings (pluginSettings: _*)
//
//lazy val condor = OsgiProject(pluginDir, "org.openmole.plugin.environment.condor", imports = Seq("*")) dependsOn(openmoleDSL, batch, gridscale, ssh) settings
// (libraryDependencies += Libraries.gridscaleCondor) settings (pluginSettings: _*)
//
//lazy val slurm = OsgiProject(pluginDir, "org.openmole.plugin.environment.slurm", imports = Seq("*")) dependsOn(openmoleDSL, batch, gridscale, ssh) settings
// (libraryDependencies += Libraries.gridscaleSLURM) settings (pluginSettings: _*)
//
lazy val ssh = OsgiProject(pluginDir, "org.openmole.plugin.environment.ssh", imports = Seq("*")) dependsOn(openmoleDSL, event, batch, gridscale) settings
(libraryDependencies += Libraries.gridscaleSSH) settings (pluginSettings: _*)
......@@ -473,7 +475,7 @@ lazy val care = OsgiProject(pluginDir, "org.openmole.plugin.task.care", imports
lazy val udocker = OsgiProject(pluginDir, "org.openmole.plugin.task.udocker", imports = Seq("!jawn.*", "*")) dependsOn(systemexec, container) settings(
libraryDependencies += Libraries.scalatest,
libraryDependencies += Libraries.circe,
libraryDependencies += Libraries.httpClient) settings (pluginSettings: _*)
libraryDependencies ++= Libraries.httpClient) settings (pluginSettings: _*)
lazy val r = OsgiProject(pluginDir, "org.openmole.plugin.task.r", imports = Seq("*")) dependsOn(udocker) settings (pluginSettings: _*)
......@@ -492,8 +494,7 @@ lazy val server = OsgiProject(
libraryDependencies ++= Seq(Libraries.bouncyCastle, Libraries.logback, Libraries.scalatra, Libraries.arm, Libraries.codec, Libraries.json4s), Libraries.addScalaLang) settings (defaultSettings: _*)
lazy val client = Project("org-openmole-rest-client", restDir / "client") settings(
libraryDependencies += "org.apache.httpcomponents" % "httpclient" % "4.3.5",
libraryDependencies += "org.apache.httpcomponents" % "httpmime" % "4.3.5",
libraryDependencies ++= Libraries.httpClient,
libraryDependencies += "org.json4s" %% "json4s-jackson" % Libraries.json4sVersion
) dependsOn(message, openmoleTar) settings (defaultSettings: _*)
......@@ -606,8 +607,8 @@ lazy val serverGUI = OsgiProject(guiServerDir, "org.openmole.gui.server.core") s
project,
openmoleDSL,
batch,
egi,
ssh,
//egi,
//ssh,
openmoleStream,
txtmark,
openmoleCrypto,
......@@ -626,11 +627,11 @@ def guiPluginSettings = defaultSettings ++ Seq(defaultActivator)
def guiPluginDir = guiDir / "plugins"
lazy val guiEnvironmentEGIPlugin = OsgiProject(guiPluginDir, "org.openmole.gui.plugin.authentication.egi") settings(
guiPluginSettings,
libraryDependencies += Libraries.equinoxOSGi
) dependsOn(extPluginGUIServer, extClientTool, dataGUI, workspace, egi) enablePlugins (ScalaJSPlugin)
//lazy val guiEnvironmentEGIPlugin = OsgiProject(guiPluginDir, "org.openmole.gui.plugin.authentication.egi") settings(
// guiPluginSettings,
// libraryDependencies += Libraries.equinoxOSGi
//) dependsOn(extPluginGUIServer, extClientTool, dataGUI, workspace, egi) enablePlugins (ScalaJSPlugin)
//
lazy val guiEnvironmentSSHKeyPlugin = OsgiProject(guiPluginDir, "org.openmole.gui.plugin.authentication.sshkey") settings(
guiPluginSettings,
libraryDependencies += Libraries.equinoxOSGi
......@@ -641,12 +642,12 @@ lazy val guiEnvironmentSSHLoginPlugin = OsgiProject(guiPluginDir, "org.openmole.
libraryDependencies += Libraries.equinoxOSGi
) dependsOn(extPluginGUIServer, extClientTool, dataGUI, workspace, ssh) enablePlugins (ScalaJSPlugin)
lazy val guiEnvironmentDesktopGridPlugin = OsgiProject(guiPluginDir, "org.openmole.gui.plugin.authentication.desktopgrid") settings(
guiPluginSettings,
libraryDependencies += Libraries.equinoxOSGi
) dependsOn(extPluginGUIServer, extClientTool, dataGUI, workspace, desktopgrid) enablePlugins (ScalaJSPlugin)
//lazy val guiEnvironmentDesktopGridPlugin = OsgiProject(guiPluginDir, "org.openmole.gui.plugin.authentication.desktopgrid") settings(
// guiPluginSettings,
// libraryDependencies += Libraries.equinoxOSGi
//) dependsOn(extPluginGUIServer, extClientTool, dataGUI, workspace, desktopgrid) enablePlugins (ScalaJSPlugin)
val guiPlugins = Seq(guiEnvironmentEGIPlugin, guiEnvironmentSSHLoginPlugin, guiEnvironmentSSHKeyPlugin, guiEnvironmentDesktopGridPlugin)
val guiPlugins = Seq(guiEnvironmentSSHLoginPlugin, guiEnvironmentSSHKeyPlugin) //Seq(guiEnvironmentEGIPlugin, , guiEnvironmentDesktopGridPlugin)
/* -------------------- Bin ------------------------- */
......@@ -655,21 +656,32 @@ def binDir = file("bin")
def bundleFilter(m: ModuleID, artifact: Artifact) = {
def exclude =
(m.organization != "org.openmole.library" && m.name.contains("slick")) || (m.name contains "sshj") || (m.name contains "scala-xml") || (m.name contains "protobuf")
(m.organization != "org.openmole.library" && m.name.contains("slick")) || (m.name contains "scala-xml") || (m.name contains "protobuf")
def include = (artifact.`type` == "bundle" && m.name != "osgi") ||
(m.name == "sshj") ||
m.organization == "org.bouncycastle" ||
(m.name == "httpclient-osgi") || (m.name == "httpcore-osgi") ||
(m.organization == "org.osgi" && m.name != "osgi")
include && !exclude
}
def noDependencyFilter(m: ModuleID, artifact: Artifact) = false
def rename(m: ModuleID): String =
def rename(m: ModuleID): String = {
val versionPattern = "([0-9]*)\\.([0-9]*).*".r
val revision =
m.revision match {
case versionPattern(major, minor) => s"$major.$minor"
case s => s
}
if (m.name.exists(_ == '-') == false) s"${m.organization.replaceAllLiterally(".", "-")}-${m.name}_${m.revision}.jar"
else s"${m.name}_${m.revision}.jar"
else s"${m.name}_${revision}.jar"
}
import Assembly._
......@@ -725,6 +737,7 @@ lazy val openmoleNaked =
resourcesAssemble += (assemble in launcher).value -> (assemblyPath.value / "launcher"),
resourcesAssemble ++= (Osgi.bundleDependencies in Compile).value.map(b b (assemblyPath.value / "plugins" / b.getName)),
libraryDependencies += Libraries.logging,
libraryDependencies += Libraries.osgiCompendium,
dependencyFilter := bundleFilter,
dependencyName := rename,
assemblyDependenciesPath := assemblyPath.value / "plugins",
......@@ -754,6 +767,7 @@ lazy val openmoleRuntime =
resourcesAssemble ++= (Osgi.bundleDependencies in Compile).value.map(b b (assemblyPath.value / "plugins" / b.getName)),
setExecutable ++= Seq("run.sh"),
tarName := "runtime.tar.gz",
libraryDependencies += Libraries.osgiCompendium,
libraryDependencies += Libraries.scopt,
libraryDependencies += Libraries.logging,
dependencyFilter := bundleFilter,
......@@ -761,29 +775,29 @@ lazy val openmoleRuntime =
) dependsOn (toDependencies(allCore): _*) settings (defaultSettings: _*)
lazy val daemon = OsgiProject(binDir, "org.openmole.daemon") enablePlugins (TarPlugin) settings (assemblySettings: _*) dependsOn(workflow, workflow, communication, workspace,
fileService, exception, tools, logging, desktopgrid) settings(
assemblyDependenciesPath := assemblyPath.value / "plugins",
resourcesAssemble ++= (Osgi.bundleDependencies in Compile).value.map(b b (assemblyPath.value / "plugins" / b.getName)),
resourcesAssemble += (resourceDirectory in Compile).value -> assemblyPath.value,
resourcesAssemble += (assemble in launcher).value -> (assemblyPath.value / "launcher"),
libraryDependencies ++= Seq(
Libraries.sshd,
Libraries.gridscale,
Libraries.gridscaleSSH,
Libraries.bouncyCastle,
Libraries.logging,
Libraries.scopt
),
Libraries.addScalaLang,
defaultActivator,
assemblyDependenciesPath := assemblyPath.value / "plugins",
dependencyFilter := bundleFilter,
dependencyName := rename,
setExecutable ++= Seq("openmole-daemon", "openmole-daemon.bat"),
tarName := "openmole-daemon.tar.gz",
tarInnerFolder := "openmole-daemon"
) settings (defaultSettings: _*)
//lazy val daemon = OsgiProject(binDir, "org.openmole.daemon") enablePlugins(TarPlugin) settings(assemblySettings: _*) dependsOn(workflow, workflow, communication, workspace,
// fileService, exception, tools, logging, desktopgrid) settings(
// assemblyDependenciesPath := assemblyPath.value / "plugins",
// resourcesAssemble ++= (Osgi.bundleDependencies in Compile).value.map(b ⇒ b → (assemblyPath.value / "plugins" / b.getName)),
// resourcesAssemble += (resourceDirectory in Compile).value -> assemblyPath.value,
// resourcesAssemble += (assemble in launcher).value -> (assemblyPath.value / "launcher"),
// libraryDependencies ++= Seq(
// Libraries.sshd,
// Libraries.gridscale,
// Libraries.gridscaleSSH,
// Libraries.bouncyCastle,
// Libraries.logging,
// Libraries.scopt
// ),
// Libraries.addScalaLang,
// defaultActivator,
// assemblyDependenciesPath := assemblyPath.value / "plugins",
// dependencyFilter := bundleFilter,
// dependencyName := rename,
// setExecutable ++= Seq("openmole-daemon", "openmole-daemon.bat"),
// tarName := "openmole-daemon.tar.gz",
// tarInnerFolder := "openmole-daemon"
//) settings (defaultSettings: _*)
lazy val api = Project("api", binDir / "target" / "api") settings (defaultSettings: _*) enablePlugins (ScalaUnidocPlugin) settings(
......@@ -811,8 +825,7 @@ lazy val site = crossProject.in(binDir / "org.openmole.site") settings (defaultS
lazy val siteJS = site.js
lazy val siteJVM = site.jvm dependsOn(tools, project, serializer, marketIndex) settings (
libraryDependencies += Libraries.sourceCode
) dependsOn (marketIndex)
libraryDependencies += Libraries.sourceCode) dependsOn (marketIndex)
lazy val marketIndex = Project("marketindex", binDir / "org.openmole.marketindex") settings (defaultSettings: _*) settings (
libraryDependencies += Libraries.json4s
......@@ -883,11 +896,13 @@ lazy val modules =
setExecutable ++= Seq("modules"),
resourcesAssemble += {
val bundle = OsgiKeys.bundle.value
bundle -> (assemblyPath.value / "plugins" / bundle.getName)},
bundle -> (assemblyPath.value / "plugins" / bundle.getName)
},
resourcesAssemble ++= (Osgi.bundleDependencies in Compile).value.map(b b (assemblyPath.value / "plugins" / b.getName)),
resourcesAssemble += ((resourceDirectory in Compile).value / "modules") -> (assemblyPath.value / "modules"),
resourcesAssemble += (assemble in launcher).value -> (assemblyPath.value / "launcher"),
dependencyFilter := bundleFilter) dependsOn (toDependencies(openmoleNakedDependencies): _*) dependsOn (toDependencies(openmoleDependencies): _*)
dependencyFilter := bundleFilter,
dependencyName := rename) dependsOn (toDependencies(openmoleNakedDependencies): _*) dependsOn (toDependencies(openmoleDependencies): _*)
lazy val launcher = OsgiProject(binDir, "org.openmole.launcher", imports = Seq("*"), settings = assemblySettings) settings(
......
......@@ -21,7 +21,7 @@ import java.util.zip.GZIPInputStream
import org.openmole.tool.file._
import org.openmole.tool.tar._
import fr.iscpif.gridscale.http.HTTPStorage
import gridscale.http
import org.openmole.core.context._
import org.openmole.core.expansion._
import org.openmole.core.fileservice.FileService
......@@ -42,10 +42,11 @@ package object market {
def indexURL(implicit preference: Preference, randomProvider: RandomProvider, newFile: NewFile, fileService: FileService) =
ExpandedString(preference(MarketIndex.marketIndexLocation)).from(Context("version" buildinfo.version))
def marketIndex(implicit preference: Preference, randomProvider: RandomProvider, newFile: NewFile, fileService: FileService) = HTTPStorage.download(indexURL)(Serialization.read[MarketIndex](_))
def marketIndex(implicit preference: Preference, randomProvider: RandomProvider, newFile: NewFile, fileService: FileService) =
http.get(indexURL).map(Serialization.read[MarketIndex](_)).get
def downloadEntry(entry: MarketIndexEntry, path: File) = try {
HTTPStorage.download(entry.url) { is
http.getStream(entry.url) { is
val tis = new TarInputStream(new GZIPInputStream(is))
try tis.extract(path)
finally tis.close
......
......@@ -20,8 +20,7 @@ package org.openmole.core
import java.io.File
import java.nio.file.FileAlreadyExistsException
import fr.iscpif.gridscale.storage._
import fr.iscpif.gridscale.http.HTTPStorage
import gridscale.http
import org.openmole.core.pluginmanager.PluginManager
import org.openmole.core.workspace.{ NewFile, Workspace }
import org.openmole.tool.file._
......@@ -50,8 +49,8 @@ package object module {
import org.json4s.jackson.Serialization
implicit val formats = Serialization.formats(NoTypeHints)
def modules(url: String) = HTTPStorage.download(url)(Serialization.read[Seq[Module]](_))
def selectableModules(url: String) = modules(url).map(m SelectableModule(Storage.parent(url).get, m))
def modules(url: String) = http.get(url).map(Serialization.read[Seq[Module]](_)).get
def selectableModules(url: String) = modules(url).map(m SelectableModule(gridscale.RemotePath.parent(url).get, m))
case class SelectableModule(baseURL: String, module: Module)
......@@ -61,8 +60,8 @@ package object module {
val hashes = downloadableComponents.map(_.component.hash).distinct.toSet -- PluginManager.bundleHashes.map(_.toString)
val files = downloadableComponents.filter(c hashes.contains(c.component.hash)).map {
c
val f = dir / Storage.name(c.component.location)
HTTPStorage.download(Storage.child(c.baseURL, c.component.location))(_.copy(f))
val f = dir / gridscale.RemotePath.name(c.component.location)
http.getStream(gridscale.RemotePath.child(c.baseURL, c.component.location))(_.copy(f))
f
}
addPluginsFiles(files, true)
......
......@@ -53,11 +53,11 @@ sealed trait Environment <: Name {
private[execution] val _done = new AtomicLong(0L)
private[execution] val _failed = new AtomicLong(0L)
implicit def preference: Preference
implicit def eventDispatcher: EventDispatcher
def eventDispatcher: EventDispatcher
def exceptions: Int
private lazy val _errors = new SlidingList[ExceptionEvent]
def error(e: ExceptionEvent) = _errors.put(e, preference(maxExceptionsLog))
def error(e: ExceptionEvent) = _errors.put(e, exceptions)
def errors: List[ExceptionEvent] = _errors.elements
def clearErrors: List[ExceptionEvent] = _errors.clear()
......@@ -95,6 +95,7 @@ class LocalEnvironment(
val pool = Cache(new ExecutorPool(nbThreads, WeakReference(this), threadProvider))
def nbJobInQueue = pool().waiting
def exceptions = preference(Environment.maxExceptionsLog)
def submit(job: Job, executionContext: TaskExecutionContext): Unit =
submit(new LocalExecutionJob(executionContext, job.moleJobs, Some(job.moleExecution)))
......
......@@ -18,7 +18,6 @@ import java.io._
import java.nio.file._
import java.util.concurrent.atomic.AtomicReference
import fr.iscpif.gridscale.http.HTTPStorage
import org.openmole.core.authentication.AuthenticationStore
import org.openmole.core.fileservice.FileService
import org.openmole.core.market.{ MarketIndex, MarketIndexEntry }
......
......@@ -21,7 +21,7 @@ import org.openmole.core.pluginmanager.PluginInfo
import org.openmole.core.preference.ConfigurationInfo
import org.openmole.core.replication.ReplicaCatalog
import org.openmole.plugin.environment.batch.environment.BatchEnvironment
import org.openmole.plugin.environment.batch.storage.{ Storage, StorageService }
import org.openmole.plugin.environment.batch.storage.{ StorageInterface, StorageService }
import org.osgi.framework.{ BundleActivator, BundleContext }
class Activator extends BundleActivator {
......
......@@ -57,17 +57,17 @@ object BatchEnvironment extends Logger {
def id: Long
}
case class BeginUpload(id: Long, file: File, path: String, storage: StorageService) extends Event[BatchEnvironment] with Transfer
case class EndUpload(id: Long, file: File, path: String, storage: StorageService, exception: Option[Throwable]) extends Event[BatchEnvironment] with Transfer {
case class BeginUpload(id: Long, file: File, path: String, storage: StorageService[_]) extends Event[BatchEnvironment] with Transfer
case class EndUpload(id: Long, file: File, path: String, storage: StorageService[_], exception: Option[Throwable]) extends Event[BatchEnvironment] with Transfer {
def success = exception.isEmpty
}
case class BeginDownload(id: Long, file: File, path: String, storage: StorageService) extends Event[BatchEnvironment] with Transfer
case class EndDownload(id: Long, file: File, path: String, storage: StorageService, exception: Option[Throwable]) extends Event[BatchEnvironment] with Transfer {
case class BeginDownload(id: Long, file: File, path: String, storage: StorageService[_]) extends Event[BatchEnvironment] with Transfer
case class EndDownload(id: Long, file: File, path: String, storage: StorageService[_], exception: Option[Throwable]) extends Event[BatchEnvironment] with Transfer {
def success = exception.isEmpty
}
def signalUpload[T](id: Long, upload: T, file: File, path: String, storage: StorageService)(implicit eventDispatcher: EventDispatcher): T = {
def signalUpload[T](id: Long, upload: T, file: File, path: String, storage: StorageService[_])(implicit eventDispatcher: EventDispatcher): T = {
eventDispatcher.trigger(storage.environment, BeginUpload(id, file, path, storage))
val res =
try upload
......@@ -80,7 +80,7 @@ object BatchEnvironment extends Logger {
res
}
def signalDownload[T](id: Long, download: T, path: String, storage: StorageService, file: File)(implicit eventDispatcher: EventDispatcher): T = {
def signalDownload[T](id: Long, download: T, path: String, storage: StorageService[_], file: File)(implicit eventDispatcher: EventDispatcher): T = {
eventDispatcher.trigger(storage.environment, BeginDownload(id, file, path, storage))
val res =
try download
......@@ -149,19 +149,26 @@ object BatchEnvironment extends Logger {
implicit val eventDispatcher: EventDispatcher
)
def trySelectSingleStorage(s: StorageService[_]) =
BatchService.tryGetToken(s.usageControl).map(t (s, t))
def trySelectSingleJobService(jobService: BatchJobService[_]) =
BatchService.tryGetToken(jobService.usageControl).map(t (jobService, t))
}
trait BatchEnvironment extends SubmissionEnvironment { env
type SS <: StorageService
type JS <: JobService
implicit val services: BatchEnvironment.Services
implicit def preference = services.preference
//implicit def preference = services.preference
implicit def eventDispatcher = services.eventDispatcher