Commit 6e40ad8a authored by Romain Reuillon's avatar Romain Reuillon
Browse files

[Plugin] feat: port download CA certificates with new gridscale version

parent 8af44f22
......@@ -332,7 +332,7 @@ lazy val sourceCode = OsgiProject(dir, "sourcecode") settings (
def gridscaleVersion = "2.0-SNAPSHOT"
lazy val gridscale = OsgiProject(dir, "gridscale", imports = Seq("*")) settings (
lazy val gridscale = OsgiProject(dir, "gridscale", imports = Seq("*"), exports = Seq("gridscale.*", "enumeratum.*")) settings (
libraryDependencies += "fr.iscpif.gridscale" %% "gridscale" % gridscaleVersion,
version := gridscaleVersion
) settings(settings: _*) dependsOn(freedsl)
......@@ -364,6 +364,21 @@ lazy val gridscaleOAR = OsgiProject(dir, "gridscale.oar", imports = Seq("*")) se
) settings(settings: _*) dependsOn(gridscale, gridscaleCluster)
lazy val gridscaleEGI = OsgiProject(dir, "gridscale.egi", imports = Seq("*")) settings (
libraryDependencies += "fr.iscpif.gridscale" %% "egi" % gridscaleVersion,
version := gridscaleVersion
) settings(settings: _*) dependsOn(gridscale, gridscaleHTTP)
lazy val gridscaleDIRAC = OsgiProject(dir, "gridscale.dirac", imports = Seq("*")) settings (
libraryDependencies += "fr.iscpif.gridscale" %% "dirac" % gridscaleVersion,
version := gridscaleVersion
) settings(settings: _*) dependsOn(gridscale, gridscaleHTTP)
lazy val gridscaleWebDAV = OsgiProject(dir, "gridscale.webdav", imports = Seq("*")) settings (
libraryDependencies += "fr.iscpif.gridscale" %% "webdav" % gridscaleVersion,
version := gridscaleVersion
) settings(settings: _*) dependsOn(gridscale, gridscaleHTTP)
lazy val jzlib = OsgiProject(dir, "com.jcraft.jzlib", imports = Seq("*")) settings (
libraryDependencies += "com.jcraft" % "jzlib" % "1.1.3",
version := "1.1.3"
......
......@@ -347,7 +347,7 @@ lazy val boundsDomain = OsgiProject(pluginDir, "org.openmole.plugin.domain.bound
/* Environment */
def allEnvironment = Seq(batch, gridscale, ssh, oar) //, desktopgrid, egi, gridscale, pbs, sge, condor, slurm, ssh)
def allEnvironment = Seq(batch, gridscale, ssh, oar, egi) //, desktopgrid, egi, gridscale, pbs, sge, condor, slurm, ssh)
lazy val batch = OsgiProject(pluginDir, "org.openmole.plugin.environment.batch", imports = Seq("*")) dependsOn(
workflow, workspace, tools, event, replication, exception,
......@@ -373,13 +373,9 @@ lazy val oar = OsgiProject(pluginDir, "org.openmole.plugin.environment.oar", imp
// batch, sftpserver, gridscale
//) settings (pluginSettings: _*)
//
//lazy val egi = OsgiProject(pluginDir, "org.openmole.plugin.environment.egi", imports = Seq("!org.apache.http.*", "!fr.iscpif.gridscale.libraries.srmstub", "!fr.iscpif.gridscale.libraries.lbstub", "!fr.iscpif.gridscale.libraries.wmsstub", "!com.google.common.cache", "*")) dependsOn(openmoleDSL,
// batch,
// workspace, fileService, gridscale) settings (
// libraryDependencies ++= Seq(Libraries.gridscaleGlite, Libraries.gridscaleHTTP),
// Libraries.addScalaLang,
// ) settings (pluginSettings: _*)
//
lazy val egi = OsgiProject(pluginDir, "org.openmole.plugin.environment.egi") dependsOn(openmoleDSL, batch, workspace, fileService, gridscale) settings (
libraryDependencies ++= Libraries.gridscaleEGI, Libraries.addScalaLang) settings (pluginSettings: _*)
lazy val gridscale = OsgiProject(pluginDir, "org.openmole.plugin.environment.gridscale", imports = Seq("*")) settings(
libraryDependencies += Libraries.gridscaleLocal)dependsOn(openmoleDSL, tools, batch, exception) settings (pluginSettings: _*)
......@@ -685,7 +681,6 @@ def bundleFilter(m: ModuleID, artifact: Artifact) = {
(m.name == "httpclient-osgi") || (m.name == "httpcore-osgi") ||
(m.organization == "org.osgi" && m.name != "osgi")
include && !exclude
}
......@@ -700,8 +695,7 @@ def rename(m: ModuleID): String = {
case s => s
}
if (m.name.exists(_ == '-') == false) s"${m.organization.replaceAllLiterally(".", "-")}-${m.name}_${m.revision}.jar"
else s"${m.name}_${revision}.jar"
s"${m.organization.replaceAllLiterally(".", "-")}-${m.name}_${m.revision}.jar"
}
......
......@@ -31,8 +31,7 @@ class Activator extends BundleActivator {
PluginInfo.add(this.getClass)
ConfigurationInfo.add(
this.getClass,
ConfigurationInfo.list(EGIEnvironment) ++
ConfigurationInfo.list(DIRACEnvironment)
ConfigurationInfo.list(EGIEnvironment) //++ ConfigurationInfo.list(DIRACEnvironment)
)
}
}
/*
* Copyright (C) 13/11/12 Romain Reuillon
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.openmole.plugin.environment.egi
import org.openmole.plugin.environment.batch.control._
import org.openmole.tool.statistics.MovingAverage
object AvailabilityQuality {
def apply(_usageControl: UsageControl, _hysteresis: Int) =
new AvailabilityQuality {
val usageControl = _usageControl
val hysteresis = _hysteresis
}
}
trait AvailabilityQuality extends QualityControl with UsageControl {
val usageControl: UsageControl
def available: Int = usageControl.available
def releaseToken(token: AccessToken): Unit = usageControl.releaseToken(token)
def tryGetToken = {
val token = usageControl.tryGetToken
token match {
case Some(_) wasAvailable
case None wasNotAvailable
}
token
}
def waitAToken = usageControl.waitAToken
}
trait QualityControl {
def hysteresis: Int
def isEmpty = _successRate.isEmpty || operationTime.isEmpty
private lazy val _successRate = new MovingAverage(hysteresis)
private lazy val operationTime = new MovingAverage(hysteresis)
private lazy val _availability = new MovingAverage(hysteresis, 1.0)
def wasAvailable = _availability.put(1.0)
def wasNotAvailable = _availability.put(0.0)
def failed = _successRate.put(0.0)
def success = _successRate.put(1.0)
def successRate = _successRate.get
def time = operationTime.get
def availability = _availability.get
def quality[A](op: A): A = timed {
try {
val ret = op
success
ret
}
catch {
case e: Throwable
failed
throw e
}
}
def timed[A](op: A): A = {
val begin = System.currentTimeMillis
val a = op
operationTime.put(System.currentTimeMillis - begin)
a
}
}
///*
// * Copyright (C) 13/11/12 Romain Reuillon
// *
// * This program is free software: you can redistribute it and/or modify
// * it under the terms of the GNU Affero General Public License as published by
// * the Free Software Foundation, either version 3 of the License, or
// * (at your option) any later version.
// *
// * This program is distributed in the hope that it will be useful,
// * but WITHOUT ANY WARRANTY; without even the implied warranty of
// * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// * GNU Affero General Public License for more details.
// *
// * You should have received a copy of the GNU General Public License
// * along with this program. If not, see <http://www.gnu.org/licenses/>.
// */
//
//package org.openmole.plugin.environment.egi
//
//import org.openmole.plugin.environment.batch.control._
//import org.openmole.tool.statistics.MovingAverage
//
//object AvailabilityQuality {
// def apply(_usageControl: UsageControl, _hysteresis: Int) =
// new AvailabilityQuality {
// val usageControl = _usageControl
// val hysteresis = _hysteresis
// }
//}
//
//trait AvailabilityQuality extends QualityControl with UsageControl {
// val usageControl: UsageControl
// def available: Int = usageControl.available
// def releaseToken(token: AccessToken): Unit = usageControl.releaseToken(token)
//
// def tryGetToken = {
// val token = usageControl.tryGetToken
// token match {
// case Some(_) ⇒ wasAvailable
// case None ⇒ wasNotAvailable
// }
// token
// }
//
// def waitAToken = usageControl.waitAToken
//}
//
//trait QualityControl {
// def hysteresis: Int
//
// def isEmpty = _successRate.isEmpty || operationTime.isEmpty
//
// private lazy val _successRate = new MovingAverage(hysteresis)
// private lazy val operationTime = new MovingAverage(hysteresis)
// private lazy val _availability = new MovingAverage(hysteresis, 1.0)
//
// def wasAvailable = _availability.put(1.0)
// def wasNotAvailable = _availability.put(0.0)
//
// def failed = _successRate.put(0.0)
// def success = _successRate.put(1.0)
//
// def successRate = _successRate.get
//
// def time = operationTime.get
// def availability = _availability.get
//
// def quality[A](op: ⇒ A): A = timed {
// try {
// val ret = op
// success
// ret
// }
// catch {
// case e: Throwable ⇒
// failed
// throw e
// }
// }
//
// def timed[A](op: ⇒ A): A = {
// val begin = System.currentTimeMillis
// val a = op
// operationTime.put(System.currentTimeMillis - begin)
// a
// }
//
//}
......@@ -19,8 +19,7 @@ package org.openmole.plugin.environment.egi
import java.io.File
import fr.iscpif.gridscale.egi.{ BDII, GlobusAuthentication }
import fr.iscpif.gridscale.tools.findWorking
import gridscale.egi._
import org.openmole.core.exception.{ InternalProcessingError, UserBadDataError }
import org.openmole.core.fileservice.FileService
import org.openmole.core.replication.ReplicaCatalog
......@@ -35,82 +34,82 @@ import org.openmole.tool.logger.Logger
object BDIIStorageServers extends Logger
import org.openmole.plugin.environment.egi.BDIIStorageServers.Log._
trait BDIIStorageServers extends BatchEnvironment { env
import services._
type SS = EGIStorageService
def bdiis: Seq[BDII]
def voName: String
def debug: Boolean
def proxyCreator: () GlobusAuthentication.Proxy
def storages = _storages()
val _storages = Cache {
val webdavStorages = findWorking(bdiis, (b: BDII) b.queryWebDAVLocations(voName))
if (!webdavStorages.isEmpty) {
logger.fine("Use webdav storages:" + webdavStorages.mkString(","))
implicit def preference = services.preference
webdavStorages.map { s EGIWebDAVStorageService(s, env, voName, debug, proxyCreator) }
}
else throw new UserBadDataError("No WebDAV storage available for the VO")
}
def trySelectAStorage(usedFiles: Vector[File]) = {
import EGIEnvironment._
val sss = storages
if (sss.isEmpty) throw new InternalProcessingError("No storage service available for the environment.")
val nonEmpty = sss.filter(!_.usageControl.isEmpty)
case class FileInfo(size: Long, hash: String)
val usedFilesInfo = usedFiles.map { f f FileInfo(f.size, fileService.hash(f).toString) }.toMap
val totalFileSize = usedFilesInfo.values.toSeq.map(_.size).sum
val onStorage = replicaCatalog.forHashes(usedFilesInfo.values.toVector.map(_.hash), sss.map(_.id)).groupBy(_.storage)
def minOption(v: Seq[Double]) = if (v.isEmpty) None else Some(v.min)
def maxOption(v: Seq[Double]) = if (v.isEmpty) None else Some(v.max)
val times = sss.flatMap(_.usageControl.time)
val maxTime = maxOption(times)
val minTime = minOption(times)
val availablities = nonEmpty.flatMap(_.usageControl.availability)
val maxAvailability = maxOption(availablities)
val minAvailability = minOption(availablities)
def rate(ss: EGIStorageService) = {
val sizesOnStorage = usedFilesInfo.filter { case (_, info) onStorage.getOrElse(ss.id, Set.empty).exists(_.hash == info.hash) }.values.map { _.size }
val sizeOnStorage = sizesOnStorage.sum
val sizeFactor = if (totalFileSize != 0) sizeOnStorage.toDouble / totalFileSize else 0.0
val timeFactor =
(minTime, maxTime, ss.usageControl.time) match {
case (Some(minTime), Some(maxTime), Some(time)) if (maxTime > minTime) 0.0 - time.normalize(minTime, maxTime)
case _ 0.0
}
val availabilityFactor =
(minAvailability, maxAvailability, ss.usageControl.availability) match {
case (Some(minAvailability), Some(maxAvailability), Some(availability)) if (maxAvailability > minAvailability) 0.0 - availability.normalize(minAvailability, maxAvailability)
case _ 0.0
}
math.pow(
preference(StorageSizeFactor) * sizeFactor +
preference(StorageTimeFactor) * timeFactor +
preference(StorageAvailabilityFactor) * availabilityFactor +
preference(StorageSuccessRateFactor) * ss.usageControl.successRate.getOrElse(0.0),
preference(StorageFitnessPower)
)
}
implicit def preference = services.preference
select(sss.toList, rate)
}
}
//
//trait BDIIStorageServers extends BatchEnvironment { env ⇒
// import services._
//
// type SS = EGIStorageService
//
// def bdiis: Seq[BDIIServer]
// def voName: String
// def debug: Boolean
// def proxyCreator: () ⇒ GlobusAuthentication.Proxy
//
// def storages = _storages()
// val _storages = Cache {
// val webdavStorages = findWorking(bdiis, (b: BDII) ⇒ b.queryWebDAVLocations(voName))
// if (!webdavStorages.isEmpty) {
// logger.fine("Use webdav storages:" + webdavStorages.mkString(","))
// implicit def preference = services.preference
// webdavStorages.map { s ⇒ EGIWebDAVStorageService(s, env, voName, debug, proxyCreator) }
// }
// else throw new UserBadDataError("No WebDAV storage available for the VO")
// }
//
// def trySelectAStorage(usedFiles: Vector[File]) = {
// import EGIEnvironment._
//
// val sss = storages
// if (sss.isEmpty) throw new InternalProcessingError("No storage service available for the environment.")
//
// val nonEmpty = sss.filter(!_.usageControl.isEmpty)
//
// case class FileInfo(size: Long, hash: String)
//
// val usedFilesInfo = usedFiles.map { f ⇒ f → FileInfo(f.size, fileService.hash(f).toString) }.toMap
// val totalFileSize = usedFilesInfo.values.toSeq.map(_.size).sum
// val onStorage = replicaCatalog.forHashes(usedFilesInfo.values.toVector.map(_.hash), sss.map(_.id)).groupBy(_.storage)
//
// def minOption(v: Seq[Double]) = if (v.isEmpty) None else Some(v.min)
// def maxOption(v: Seq[Double]) = if (v.isEmpty) None else Some(v.max)
//
// val times = sss.flatMap(_.usageControl.time)
// val maxTime = maxOption(times)
// val minTime = minOption(times)
//
// val availablities = nonEmpty.flatMap(_.usageControl.availability)
// val maxAvailability = maxOption(availablities)
// val minAvailability = minOption(availablities)
//
// def rate(ss: EGIStorageService) = {
// val sizesOnStorage = usedFilesInfo.filter { case (_, info) ⇒ onStorage.getOrElse(ss.id, Set.empty).exists(_.hash == info.hash) }.values.map { _.size }
// val sizeOnStorage = sizesOnStorage.sum
//
// val sizeFactor = if (totalFileSize != 0) sizeOnStorage.toDouble / totalFileSize else 0.0
//
// val timeFactor =
// (minTime, maxTime, ss.usageControl.time) match {
// case (Some(minTime), Some(maxTime), Some(time)) if (maxTime > minTime) ⇒ 0.0 - time.normalize(minTime, maxTime)
// case _ ⇒ 0.0
// }
//
// val availabilityFactor =
// (minAvailability, maxAvailability, ss.usageControl.availability) match {
// case (Some(minAvailability), Some(maxAvailability), Some(availability)) if (maxAvailability > minAvailability) ⇒ 0.0 - availability.normalize(minAvailability, maxAvailability)
// case _ ⇒ 0.0
// }
//
// math.pow(
// preference(StorageSizeFactor) * sizeFactor +
// preference(StorageTimeFactor) * timeFactor +
// preference(StorageAvailabilityFactor) * availabilityFactor +
// preference(StorageSuccessRateFactor) * ss.usageControl.successRate.getOrElse(0.0),
// preference(StorageFitnessPower)
// )
// }
//
// implicit def preference = services.preference
// select(sss.toList, rate)
// }
//
//}
/*
* Copyright (C) 2014 Romain Reuillon
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.openmole.plugin.environment.egi
import java.net.URI
import org.openmole.core.workspace.Workspace
import squants.time.Time
trait CpCommands {
def upload(from: String, to: URI): String
def download(from: URI, to: String): String
}
case class Curl(voName: String, debug: Boolean, timeOut: Time) extends CpCommands {
@transient lazy val curl =
s"curl ${if (debug) "--verbose" else ""} --connect-timeout ${timeOut.toSeconds.toInt.toString} --max-time ${timeOut.toSeconds.toInt.toString} --cert $$X509_USER_PROXY --key $$X509_USER_PROXY --cacert $$X509_USER_PROXY --capath $$X509_CERT_DIR -f "
def upload(from: String, to: URI) = s"$curl -T $from -L $to"
def download(from: URI, to: String) = s"$curl -L $from -o $to"
}
\ No newline at end of file
///*
// * Copyright (C) 2014 Romain Reuillon
// *
// * This program is free software: you can redistribute it and/or modify
// * it under the terms of the GNU Affero General Public License as published by
// * the Free Software Foundation, either version 3 of the License, or
// * (at your option) any later version.
// *
// * This program is distributed in the hope that it will be useful,
// * but WITHOUT ANY WARRANTY; without even the implied warranty of
// * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// * GNU General Public License for more details.
// *
// * You should have received a copy of the GNU General Public License
// * along with this program. If not, see <http://www.gnu.org/licenses/>.
// */
//
//package org.openmole.plugin.environment.egi
//
//import java.net.URI
//
//import org.openmole.core.workspace.Workspace
//import squants.time.Time
//
//trait CpCommands {
// def upload(from: String, to: URI): String
// def download(from: URI, to: String): String
//}
//
//case class Curl(voName: String, debug: Boolean, timeOut: Time) extends CpCommands {
// @transient lazy val curl =
// s"curl ${if (debug) "--verbose" else ""} --connect-timeout ${timeOut.toSeconds.toInt.toString} --max-time ${timeOut.toSeconds.toInt.toString} --cert $$X509_USER_PROXY --key $$X509_USER_PROXY --cacert $$X509_USER_PROXY --capath $$X509_CERT_DIR -f "
//
// def upload(from: String, to: URI) = s"$curl -T $from -L $to"
// def download(from: URI, to: String) = s"$curl -L $from -o $to"
//}
\ No newline at end of file
/*
* Copyright (C) 10/06/13 Romain Reuillon
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.openmole.plugin.environment.egi
import fr.iscpif.gridscale.authentication._
import org.openmole.core.exception.UserBadDataError
import org.openmole.tool.crypto.Cypher
object DIRACAuthentication {
def initialise(a: EGIAuthentication)(implicit cypher: Cypher): P12Authentication =
a match {
case a: P12Certificate P12Authentication(a.certificate, password = a.password)
case _ throw new UserBadDataError(s"Wrong authentication type ${a.getClass.getName} DIRAC only supports P12 authentication")
}
}
///*
// * Copyright (C) 10/06/13 Romain Reuillon
// *
// * This program is free software: you can redistribute it and/or modify
// * it under the terms of the GNU Affero General Public License as published by
// * the Free Software Foundation, either version 3 of the License, or
// * (at your option) any later version.