Commit b5769181 authored by Romain Reuillon's avatar Romain Reuillon

[Plugin] enh: implement clearContainerCache in GAMATask

parent 19de59e0
...@@ -235,7 +235,7 @@ lazy val mgo = OsgiProject(dir, "mgo", exports = Seq("mgo.*", "freestyle.*"), im ...@@ -235,7 +235,7 @@ lazy val mgo = OsgiProject(dir, "mgo", exports = Seq("mgo.*", "freestyle.*"), im
version := mgoVersion) dependsOn(monocle, math, cats, squants) settings(settings: _*) version := mgoVersion) dependsOn(monocle, math, cats, squants) settings(settings: _*)
lazy val containerVersion = "1.10" lazy val containerVersion = "1.11"
lazy val container = OsgiProject(dir, "container", exports = Seq("container.*"), imports = Seq("!better.*", "!javax.xml.*", "!scala.meta.*", "!sun.misc.*", "!com.github.luben.*", "!org.apache.avalon.*", "!org.apache.log.*", "!org.brotli.dec.*", "!javax.*","*"), privatePackages = Seq("!scala.*", "!monocle.*", "!org.apache.commons.math3.*", "!cats.*", "!squants.*", "!scalaz.*", "!io.circe.*", "!shapeless.*", "*")) settings( lazy val container = OsgiProject(dir, "container", exports = Seq("container.*"), imports = Seq("!better.*", "!javax.xml.*", "!scala.meta.*", "!sun.misc.*", "!com.github.luben.*", "!org.apache.avalon.*", "!org.apache.log.*", "!org.brotli.dec.*", "!javax.*","*"), privatePackages = Seq("!scala.*", "!monocle.*", "!org.apache.commons.math3.*", "!cats.*", "!squants.*", "!scalaz.*", "!io.circe.*", "!shapeless.*", "*")) settings(
libraryDependencies += "org.openmole" %% "container" % containerVersion, libraryDependencies += "org.openmole" %% "container" % containerVersion,
......
...@@ -69,11 +69,7 @@ object ContainerTask { ...@@ -69,11 +69,7 @@ object ContainerTask {
import _root_.container._ import _root_.container._
ImageDownloader.downloadContainerImage( ImageDownloader.downloadContainerImage(
_root_.container.RegistryImage( DockerImage.toRegistryImage(image),
imageName = image.image,
tag = image.tag,
registry = image.registry
),
repository, repository,
timeout = preference(RegistryTimeout), timeout = preference(RegistryTimeout),
retry = Some(preference(RegistryRetryOnError)), retry = Some(preference(RegistryRetryOnError)),
...@@ -162,6 +158,7 @@ object ContainerTask { ...@@ -162,6 +158,7 @@ object ContainerTask {
stdOut: OptionalArgument[Val[String]] = None, stdOut: OptionalArgument[Val[String]] = None,
stdErr: OptionalArgument[Val[String]] = None, stdErr: OptionalArgument[Val[String]] = None,
reuseContainer: Boolean = true, reuseContainer: Boolean = true,
clearCache: Boolean = false,
containerPoolKey: CacheKey[WithInstance[_root_.container.FlatImage]] = CacheKey())(implicit name: sourcecode.Name, definitionScope: DefinitionScope, tmpDirectory: TmpDirectory, networkService: NetworkService, workspace: Workspace, threadProvider: ThreadProvider, preference: Preference, outputRedirection: OutputRedirection, serializerService: SerializerService) = { containerPoolKey: CacheKey[WithInstance[_root_.container.FlatImage]] = CacheKey())(implicit name: sourcecode.Name, definitionScope: DefinitionScope, tmpDirectory: TmpDirectory, networkService: NetworkService, workspace: Workspace, threadProvider: ThreadProvider, preference: Preference, outputRedirection: OutputRedirection, serializerService: SerializerService) = {
new ContainerTask( new ContainerTask(
containerSystem, containerSystem,
...@@ -184,7 +181,7 @@ object ContainerTask { ...@@ -184,7 +181,7 @@ object ContainerTask {
) )
} }
def prepare(containerSystem: ContainerSystem, image: ContainerImage, install: Seq[String], volumes: Seq[(String, String)] = Seq.empty, errorDetail: Int Option[String] = _ None)(implicit tmpDirectory: TmpDirectory, serializerService: SerializerService, outputRedirection: OutputRedirection, networkService: NetworkService, threadProvider: ThreadProvider, preference: Preference, workspace: Workspace) = { def prepare(containerSystem: ContainerSystem, image: ContainerImage, install: Seq[String], volumes: Seq[(String, String)] = Seq.empty, errorDetail: Int Option[String] = _ None, clearCache: Boolean = false)(implicit tmpDirectory: TmpDirectory, serializerService: SerializerService, outputRedirection: OutputRedirection, networkService: NetworkService, threadProvider: ThreadProvider, preference: Preference, workspace: Workspace) = {
def cacheId(image: ContainerImage): Seq[String] = def cacheId(image: ContainerImage): Seq[String] =
image match { image match {
case image: DockerImage Seq(image.image, image.tag, image.registry) case image: DockerImage Seq(image.image, image.tag, image.registry)
...@@ -198,10 +195,16 @@ object ContainerTask { ...@@ -198,10 +195,16 @@ object ContainerTask {
val serializedFlatImage = cacheDirectory / "flatimage.bin" val serializedFlatImage = cacheDirectory / "flatimage.bin"
cacheDirectory.withLockInDirectory { cacheDirectory.withLockInDirectory {
val containerDirectory = cacheDirectory / "fs"
if (clearCache) {
serializedFlatImage.delete
containerDirectory.recursiveDelete
}
if (serializedFlatImage.exists) serializerService.deserialize[_root_.container.FlatImage](serializedFlatImage) if (serializedFlatImage.exists) serializerService.deserialize[_root_.container.FlatImage](serializedFlatImage)
else { else {
val containerDirectory = cacheDirectory / "fs" val img = localImage(image, containerDirectory, clearCache = clearCache)
val img = localImage(image, containerDirectory)
val installedImage = executeInstall(containerSystem, img, install, volumes = volumes, errorDetail = errorDetail) val installedImage = executeInstall(containerSystem, img, install, volumes = volumes, errorDetail = errorDetail)
serializerService.serialize(installedImage, serializedFlatImage) serializerService.serialize(installedImage, serializedFlatImage)
installedImage installedImage
...@@ -217,9 +220,10 @@ object ContainerTask { ...@@ -217,9 +220,10 @@ object ContainerTask {
image image
} }
def localImage(image: ContainerImage, containerDirectory: File)(implicit networkService: NetworkService, workspace: Workspace, threadProvider: ThreadProvider, preference: Preference, tmpDirectory: TmpDirectory) = def localImage(image: ContainerImage, containerDirectory: File, clearCache: Boolean)(implicit networkService: NetworkService, workspace: Workspace, threadProvider: ThreadProvider, preference: Preference, tmpDirectory: TmpDirectory) =
image match { image match {
case image: DockerImage case image: DockerImage
if (clearCache) _root_.container.ImageDownloader.imageDirectory(repositoryDirectory(workspace), DockerImage.toRegistryImage(image)).recursiveDelete
val savedImage = downloadImage(image, repositoryDirectory(workspace)) val savedImage = downloadImage(image, repositoryDirectory(workspace))
_root_.container.ImageBuilder.flattenImage(savedImage, containerDirectory) _root_.container.ImageBuilder.flattenImage(savedImage, containerDirectory)
case image: SavedDockerImage case image: SavedDockerImage
......
...@@ -60,6 +60,15 @@ package container { ...@@ -60,6 +60,15 @@ package container {
} }
object DockerImage {
def toRegistryImage(image: DockerImage) =
_root_.container.RegistryImage(
name = image.image,
tag = image.tag,
registry = image.registry
)
}
sealed trait ContainerImage sealed trait ContainerImage
case class DockerImage(image: String, tag: String = "latest", registry: String = "https://registry-1.docker.io") extends ContainerImage case class DockerImage(image: String, tag: String = "latest", registry: String = "https://registry-1.docker.io") extends ContainerImage
case class SavedDockerImage(file: java.io.File, compressed: Boolean) extends ContainerImage case class SavedDockerImage(file: java.io.File, compressed: Boolean) extends ContainerImage
......
...@@ -46,7 +46,8 @@ object GAMATask { ...@@ -46,7 +46,8 @@ object GAMATask {
experiment: String, experiment: String,
install: Seq[String], install: Seq[String],
installContainerSystem: ContainerSystem, installContainerSystem: ContainerSystem,
version: String)(implicit tmpDirectory: TmpDirectory, serializerService: SerializerService, outputRedirection: OutputRedirection, networkService: NetworkService, threadProvider: ThreadProvider, preference: Preference, _workspace: Workspace) = { version: String,
clearCache: Boolean)(implicit tmpDirectory: TmpDirectory, serializerService: SerializerService, outputRedirection: OutputRedirection, networkService: NetworkService, threadProvider: ThreadProvider, preference: Preference, _workspace: Workspace) = {
val (modelName, volumesValue) = volumes(workspace, model) val (modelName, volumesValue) = volumes(workspace, model)
...@@ -59,7 +60,7 @@ object GAMATask { ...@@ -59,7 +60,7 @@ object GAMATask {
case _ => None case _ => None
} }
ContainerTask.prepare(installContainerSystem, gamaImage(version), installCommands, volumesValue.map { case (lv, cv) lv.getAbsolutePath -> cv }, error) ContainerTask.prepare(installContainerSystem, gamaImage(version), installCommands, volumesValue.map { case (lv, cv) lv.getAbsolutePath -> cv }, error, clearCache = clearCache)
} }
def apply( def apply(
...@@ -78,6 +79,7 @@ object GAMATask { ...@@ -78,6 +79,7 @@ object GAMATask {
environmentVariables: Seq[EnvironmentVariable] = Vector.empty, environmentVariables: Seq[EnvironmentVariable] = Vector.empty,
hostFiles: Seq[HostFile] = Vector.empty, hostFiles: Seq[HostFile] = Vector.empty,
workDirectory: OptionalArgument[String] = None, workDirectory: OptionalArgument[String] = None,
clearContainerCache: Boolean = false,
containerSystem: ContainerSystem = ContainerSystem.default, containerSystem: ContainerSystem = ContainerSystem.default,
installContainerSystem: ContainerSystem = ContainerSystem.default)(implicit name: sourcecode.Name, definitionScope: DefinitionScope, newFile: TmpDirectory, _workspace: Workspace, preference: Preference, fileService: FileService, threadProvider: ThreadProvider, outputRedirection: OutputRedirection, networkService: NetworkService, serializerService: SerializerService): GAMATask = { installContainerSystem: ContainerSystem = ContainerSystem.default)(implicit name: sourcecode.Name, definitionScope: DefinitionScope, newFile: TmpDirectory, _workspace: Workspace, preference: Preference, fileService: FileService, threadProvider: ThreadProvider, outputRedirection: OutputRedirection, networkService: NetworkService, serializerService: SerializerService): GAMATask = {
...@@ -88,7 +90,7 @@ object GAMATask { ...@@ -88,7 +90,7 @@ object GAMATask {
case _ case _
} }
val preparedImage = prepare(workspace, model, experiment, install, installContainerSystem, version) val preparedImage = prepare(workspace, model, experiment, install, installContainerSystem, version, clearCache = clearContainerCache)
GAMATask( GAMATask(
workspace = workspace, workspace = workspace,
......
...@@ -8,7 +8,7 @@ object Libraries { ...@@ -8,7 +8,7 @@ object Libraries {
lazy val gridscaleVersion = "2.28" lazy val gridscaleVersion = "2.28"
lazy val sshjVersion = "0.27.0" lazy val sshjVersion = "0.27.0"
lazy val containerVersion = "1.10" lazy val containerVersion = "1.11"
lazy val mgoVersion = "3.43" lazy val mgoVersion = "3.43"
lazy val bouncyCastleVersion = "1.64" lazy val bouncyCastleVersion = "1.64"
lazy val d3Version = "3.5.12" lazy val d3Version = "3.5.12"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment