Commit 26be5c6e authored by Romain Reuillon's avatar Romain Reuillon
Browse files

[Lib + Plugin]: prepare for 2.13 and improve webdav init for egi

parent 2d5c2121
Pipeline #435 failed with stage
in 4 minutes and 36 seconds
......@@ -43,7 +43,7 @@ lazy val shapeless = OsgiProject(dir, "com.chuusai.shapeless", exports = Seq("s
version := shapelessVersion
) settings(settings: _*)
lazy val circeVersion = "0.12.2"
lazy val circeVersion = "0.12.3"
lazy val circe = OsgiProject(dir, "io.circe",
exports = Seq("io.circe.*", "!cats.*", "!scala.*", "!shapeless.*"),
privatePackages = Seq("org.typelevel.jawn.*"),
......@@ -51,7 +51,6 @@ lazy val circe = OsgiProject(dir, "io.circe",
libraryDependencies ++= Seq(
"io.circe" %% "circe-core",
"io.circe" %% "circe-generic",
"io.circe" %% "circe-generic-extras",
"io.circe" %% "circe-parser"
).map(_ % circeVersion),
version := circeVersion) settings(settings: _*) dependsOn(shapeless)
......@@ -210,7 +209,7 @@ lazy val closureCompilerVersion = "v20130603"
lazy val closureCompiler = OsgiProject(dir, "closure-compiler", exports = Seq("com.google.javascript.*")) settings(
libraryDependencies += "com.google.javascript" % "closure-compiler" % closureCompilerVersion, version := closureCompilerVersion) settings(settings: _*)
lazy val catsVersion = "2.0.0"
lazy val catsVersion = "2.1.0"
lazy val cats =
OsgiProject(dir, "cats") settings (
libraryDependencies += "org.typelevel" %% "cats-core" % catsVersion,
......@@ -218,7 +217,7 @@ lazy val cats =
version := catsVersion
) settings(settings: _*)
lazy val squantsVersion = "1.5.0"
lazy val squantsVersion = "1.6.0"
lazy val squants =
OsgiProject(dir, "squants") settings (
......@@ -227,14 +226,14 @@ lazy val squants =
) settings(settings: _*)
lazy val mgoVersion = "3.41"
lazy val mgoVersion = "3.42"
lazy val mgo = OsgiProject(dir, "mgo", exports = Seq("mgo.*", "freestyle.*"), imports = Seq("!better.*", "!javax.xml.*", "!scala.meta.*", "!sun.misc.*", "*"), privatePackages = Seq("!scala.*", "!monocle.*", "!org.apache.commons.math3.*", "!cats.*", "!squants.*", "!scalaz.*", "*")) settings(
libraryDependencies += "org.openmole" %% "mgo" % mgoVersion,
version := mgoVersion) dependsOn(monocle, math, cats, squants) settings(settings: _*)
lazy val containerVersion = "1.4"
lazy val containerVersion = "1.5"
lazy val container = OsgiProject(dir, "container", exports = Seq("container.*"), imports = Seq("!better.*", "!javax.xml.*", "!scala.meta.*", "!sun.misc.*", "!com.github.luben.*", "!org.apache.avalon.*", "!org.apache.log.*", "!org.brotli.dec.*", "!javax.*","*"), privatePackages = Seq("!scala.*", "!monocle.*", "!org.apache.commons.math3.*", "!cats.*", "!squants.*", "!scalaz.*", "!io.circe.*", "!shapeless.*", "*")) settings(
libraryDependencies += "org.openmole" %% "container" % containerVersion,
......@@ -253,15 +252,6 @@ lazy val spatialdata = OsgiProject(dir, "org.openmole.spatialdata",
//embeddedJars := (Keys.externalDependencyClasspath in Compile).value map (_.data) filter (f=> (f.getName startsWith "gt-")) // embed geotools jars
) settings(settings: _*)
/*lazy val familyVersion = "1.3"
lazy val family = OsgiProject(dir, "fr.iscpif.family") settings(
libraryDependencies += "fr.iscpif" %% "family" % familyVersion,
version := familyVersion
) settings(settings: _*)*/
lazy val opencsv = OsgiProject(dir, "au.com.bytecode.opencsv") settings(
libraryDependencies += "net.sf.opencsv" % "opencsv" % "2.3",
version := "2.3"
......@@ -279,15 +269,10 @@ lazy val scalajHttp = OsgiProject(dir, "org.scalaj.scalaj-http") settings(
) settings(settings: _*)
lazy val scopt = OsgiProject(dir, "com.github.scopt", exports = Seq("scopt.*")) settings(
libraryDependencies += "com.github.scopt" %% "scopt" % "3.5.0",
version := "3.5.0"
libraryDependencies += "com.github.scopt" %% "scopt" % "3.7.1",
version := "3.7.1"
) settings(settings: _*)
/*lazy val scalabc = OsgiProject(dir, "fr.iscpif.scalabc", privatePackages = Seq("!scala.*", "!junit.*", "*")) settings(
libraryDependencies += "fr.iscpif" %% "scalabc" % "0.4",
version := "0.4"
) settings(settings: _*)*/
lazy val async =
OsgiProject(dir, "scala-async") settings (
libraryDependencies += "org.scala-lang.modules" %% "scala-async" % "0.9.6",
......@@ -356,24 +341,18 @@ lazy val config = OsgiProject(dir, "org.apache.commons.configuration2",
libraryDependencies += "commons-beanutils" % "commons-beanutils" % "1.9.2",
version := "2.2") settings(settings: _*) dependsOn (logging)
def sourceCodeVersion = "0.1.4"
def sourceCodeVersion = "0.1.9"
lazy val sourceCode = OsgiProject(dir, "sourcecode") settings (
libraryDependencies += "com.lihaoyi" %% "sourcecode" % sourceCodeVersion,
version := sourceCodeVersion
) settings(settings: _*)
def effectasideVersion = "0.2"
lazy val effectaside = OsgiProject(dir, "effectaside", imports = Seq("*")) settings (
libraryDependencies += "fr.iscpif.effectaside" %% "effect"% effectasideVersion,
version := effectasideVersion
)
def gridscaleVersion = "2.27"
def gridscaleVersion = "2.28"
lazy val gridscale = OsgiProject(dir, "gridscale", imports = Seq("*"), exports = Seq("gridscale.*", "enumeratum.*")) settings (
libraryDependencies += "org.openmole.gridscale" %% "gridscale" % gridscaleVersion,
version := gridscaleVersion
) settings(settings: _*) dependsOn(effectaside)
) settings(settings: _*)
lazy val gridscaleLocal = OsgiProject(dir, "gridscale.local", imports = Seq("*")) settings (
libraryDependencies += "org.openmole.gridscale" %% "local" % gridscaleVersion,
......
......@@ -50,6 +50,8 @@ class ThreadProvider(poolSize: Int) {
lazy val scheduler = Executors.newScheduledThreadPool(1, threadFactory)
lazy val taskQueue = PriorityQueue[ThreadProvider.Closure](true)
implicit lazy val executionContext: scala.concurrent.ExecutionContext = scala.concurrent.ExecutionContext.fromExecutor(pool)
var stopped = false
def stop() = synchronized {
......@@ -64,6 +66,8 @@ class ThreadProvider(poolSize: Int) {
pool.submit(new ThreadProvider.RunClosure(taskQueue))
}
def submit[T](t: T) = scala.concurrent.Future[T] { t }
def newThread(runnable: Runnable, groupName: Option[String] = None) = synchronized {
if (stopped) throw new RuntimeException("Thread provider has been stopped")
val group = groupName.map(n new ThreadGroup(parentGroup, n)).getOrElse(parentGroup)
......
......@@ -51,7 +51,7 @@ sealed class LogRange[T](val range: Range[T], val steps: FromContext[Int])(impli
import range._
def computeValues = (min |@| max |@| steps) map { (min, max, steps)
def computeValues = (min, max, steps) mapN { (min, max, steps)
val logMin: T = lg.log(min)
val logMax: T = lg.log(max)
......
......@@ -52,7 +52,7 @@ object Range {
): SizeRange[T] =
SizeRange[T](Range[T](min, max), size)
def rangeCenter[T](r: Range[T]): FromContext[T] = (r.min |@| r.max) map { (min, max)
def rangeCenter[T](r: Range[T]): FromContext[T] = (r.min, r.max) mapN { (min, max)
import r.ops._
min + ((max - min) / fromInt(2))
}
......
......@@ -403,7 +403,7 @@ abstract class BatchEnvironment extends SubmissionEnvironment { env ⇒
def clean = BatchEnvironment.registryIsEmpty(env)
lazy val registry = new ExecutionJobRegistry()
val registry = new ExecutionJobRegistry()
def jobs = ExecutionJobRegistry.executionJobs(registry)
......
......@@ -10,6 +10,7 @@ import scala.util._
import squants.time.TimeConversions._
import gridscale._
import org.openmole.core.exception.InternalProcessingError
import org.openmole.core.outputmanager.OutputManager
import org.openmole.plugin.environment.batch.environment.{ AccessControl, BatchEnvironment }
import org.openmole.plugin.environment.batch.refresh.{ JobManager, RetryAction }
import org.openmole.tool.cache.Lazy
......@@ -137,7 +138,9 @@ object HierarchicalStorageSpace extends JavaLogger {
util.Try(hierarchicalStorageInterface.makeDir(s, basePath)) match {
case util.Success(_) basePath
case util.Failure(e)
if (storageInterface.exists(s, basePath)) basePath else throw e
if (isConnectionError(e)) throw e
else if (storageInterface.exists(s, basePath)) basePath
else throw e
}
}
......
......@@ -27,6 +27,7 @@ import org.openmole.plugin.environment.ssh._
import org.openmole.tool.crypto.Cypher
import squants.Time
import squants.information._
import _root_.gridscale.effectaside
object CondorEnvironment {
......
......@@ -5,6 +5,7 @@ import org.openmole.plugin.environment.batch.environment.{ AccessControl, BatchE
import org.openmole.plugin.environment.batch.storage.{ HierarchicalStorageInterface, StorageInterface }
import org.openmole.plugin.environment.gridscale.GridScaleJobService
import org.openmole.plugin.environment.ssh.{ RuntimeInstallation, SharedStorage }
import _root_.gridscale.effectaside
class CondorJobService[S, H](
s: S,
......
......@@ -20,6 +20,7 @@ package org.openmole.plugin.environment.egi
import gridscale.egi._
import org.openmole.core.communication.storage.TransferOptions
import org.openmole.core.exception.{ InternalProcessingError, MultipleException }
import org.openmole.core.outputmanager.OutputManager
import org.openmole.core.preference.{ ConfigurationLocation, Preference }
import org.openmole.core.serializer.SerializerService
import org.openmole.core.workflow.dsl._
......@@ -37,6 +38,7 @@ import squants.time.TimeConversions._
import scala.collection.immutable.TreeSet
import scala.collection.mutable.{ HashMap, MultiMap, Set }
import scala.concurrent.{ Await, Future }
object EGIEnvironment extends JavaLogger {
......@@ -218,32 +220,41 @@ class EGIEnvironment[A: EGIAuthenticationInterface](
override def start() = {
proxyCache()
if (storages.map(_.toOption).flatten.isEmpty) throw new InternalProcessingError(s"No webdav storage is working for the VO $voName", MultipleException(storages.collect { case util.Failure(e) e }))
if (storages().map(_.toOption).flatten.isEmpty) throw new InternalProcessingError(s"No webdav storage is working for the VO $voName", MultipleException(storages().collect { case util.Failure(e) e }))
jobService
}
override def stop() = {
stopped = true
storages.map(_.toOption).flatten.foreach { case (space, storage) HierarchicalStorageSpace.clean(storage, space, background = false) }
storages().map(_.toOption).flatten.foreach { case (space, storage) HierarchicalStorageSpace.clean(storage, space, background = false) }
BatchEnvironment.waitJobKilled(this)
}
def bdiis: Seq[gridscale.egi.BDIIServer] =
bdiiURL.map(b Seq(EGIEnvironment.toBDII(new java.net.URI(b)))).getOrElse(EGIEnvironment.defaultBDIIs)
lazy val storages = {
val storages = Lazy {
val webdavStorages = findFirstWorking(bdiis) { b: BDIIServer webDAVs(b, voName) }
if (!webdavStorages.isEmpty) webdavStorages.map { location
def isConnectionError(t: Throwable) = t match {
case _: _root_.gridscale.authentication.AuthenticationException true
case _ false
}
val storage = WebDavStorage(location, AccessControl(preference(EGIEnvironment.ConnexionsByWebDAVSE)), QualityControl(preference(BatchEnvironment.QualityHysteresis)), proxyCache, env)
def storageSpace = util.Try { HierarchicalStorageSpace.create(storage, "", location, isConnectionError) }
storageSpace.map(s (s, storage))
if (!webdavStorages.isEmpty) {
webdavStorages.map { location
threadProvider.submit {
def isConnectionError(t: Throwable) = {
(t, t.getCause) match {
case (_: _root_.gridscale.authentication.AuthenticationException, _) true
case (_, _: java.net.SocketException) true
case _ false
}
}
val storage = WebDavStorage(location, AccessControl(preference(EGIEnvironment.ConnexionsByWebDAVSE)), QualityControl(preference(BatchEnvironment.QualityHysteresis)), proxyCache, env)
val storageSpace = util.Try { HierarchicalStorageSpace.create(storage, "", location, isConnectionError) }
storageSpace.map { s (s, storage) }
}
}.map(Await.result(_, scala.concurrent.duration.Duration.Inf))
}
else throw new InternalProcessingError(s"No WebDAV storage available for the VO $voName")
}
def execute(batchExecutionJob: BatchExecutionJob) = {
......@@ -252,7 +263,7 @@ class EGIEnvironment[A: EGIAuthenticationInterface](
import org.openmole.tool.file._
def selectStorage = {
val sss = storages.map(_.toOption).flatten
val sss = storages().map(_.toOption).flatten
if (sss.isEmpty) throw new InternalProcessingError("No storage service available for the environment.")
if (sss.size == 1) sss.head
......
......@@ -52,7 +52,7 @@
package org.openmole.plugin.environment.egi
import effectaside.Effect
import gridscale.effectaside.Effect
import gridscale.egi.VOMS
import org.openmole.core.communication.storage._
import org.openmole.core.workspace.TmpDirectory
......
......@@ -26,7 +26,7 @@ import org.openmole.tool.file._
object LocalStorage {
import effectaside._
import gridscale.effectaside._
import gridscale.local
def home(implicit interpreter: Effect[local.Local]) = interpreter().home()
......
......@@ -24,7 +24,7 @@ import org.openmole.tool.file._
object LogicalLinkStorage {
import effectaside._
import gridscale.effectaside._
implicit def interpreter = _root_.gridscale.local.Local()
......
......@@ -17,6 +17,7 @@
package org.openmole.plugin.environment.oar
import _root_.gridscale.effectaside
import org.openmole.core.authentication._
import org.openmole.core.workflow.dsl._
import org.openmole.core.workflow.execution._
......
......@@ -5,6 +5,7 @@ import org.openmole.plugin.environment.batch.environment.{ AccessControl, BatchE
import org.openmole.plugin.environment.batch.storage.{ HierarchicalStorageInterface, StorageInterface }
import org.openmole.plugin.environment.gridscale.GridScaleJobService
import org.openmole.plugin.environment.ssh.{ RuntimeInstallation, SharedStorage }
import _root_.gridscale.effectaside
class OARJobService[S, H](
s: S,
......
......@@ -17,6 +17,7 @@
package org.openmole.plugin.environment.pbs
import _root_.gridscale.effectaside
import org.openmole.core.authentication._
import org.openmole.core.workflow.dsl._
import org.openmole.core.workflow.execution._
......
......@@ -7,6 +7,7 @@ import org.openmole.plugin.environment.batch.storage.{ HierarchicalStorageInterf
import org.openmole.plugin.environment.gridscale.GridScaleJobService
import org.openmole.plugin.environment.pbs.PBSEnvironment.Parameters
import org.openmole.plugin.environment.ssh.{ RuntimeInstallation, SharedStorage }
import _root_.gridscale.effectaside
class PBSJobService[S, H](
s: S,
......
......@@ -17,6 +17,7 @@
package org.openmole.plugin.environment.sge
import _root_.gridscale.effectaside
import org.openmole.core.authentication._
import org.openmole.core.workflow.dsl._
import org.openmole.core.workflow.execution._
......
......@@ -5,6 +5,7 @@ import org.openmole.plugin.environment.batch.environment.{ AccessControl, BatchE
import org.openmole.plugin.environment.batch.storage.{ HierarchicalStorageInterface, StorageInterface }
import org.openmole.plugin.environment.gridscale.GridScaleJobService
import org.openmole.plugin.environment.ssh.{ RuntimeInstallation, SharedStorage }
import _root_.gridscale.effectaside
class SGEJobService[S, H](
s: S,
......
......@@ -18,6 +18,7 @@
package org.openmole.plugin.environment.slurm
import _root_.gridscale.effectaside
import org.openmole.core.authentication._
import org.openmole.core.workflow.dsl._
import org.openmole.core.workflow.execution._
......
......@@ -5,6 +5,7 @@ import org.openmole.plugin.environment.batch.environment.{ AccessControl, BatchE
import org.openmole.plugin.environment.batch.storage.{ HierarchicalStorageInterface, StorageInterface }
import org.openmole.plugin.environment.gridscale.GridScaleJobService
import org.openmole.plugin.environment.ssh.{ RuntimeInstallation, SharedStorage }
import _root_.gridscale.effectaside
class SLURMJobService[S, H](
s: S,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment