Commit d8ddbe64 authored by Romain Reuillon's avatar Romain Reuillon

[Tools] enh: implement atomic write to file

parent 2d3cbf20
......@@ -79,7 +79,7 @@ object FileSerialisation {
}
else {
val dest = newFile.newFile("fileFromArchive", ".bin")
dest.createParentDir
dest.createParentDirectory
if (exists) fromArchive.move(dest)
else dest.delete
dest
......
......@@ -44,7 +44,7 @@ object CSVHook {
val h = if (f.isEmpty) Some(headerLine) else None
if (create) f.withPrintStream(create = true) { ps csv.writeVariablesToCSV(ps, h, vs, format.arrayOnRow) }
if (create) f.atomicWithPrintStream { ps csv.writeVariablesToCSV(ps, h, vs, format.arrayOnRow) }
else f.withPrintStream(append = true, create = true) { ps csv.writeVariablesToCSV(ps, h, vs, format.arrayOnRow) }
case WritableOutput.PrintStreamValue(ps)
......@@ -57,6 +57,8 @@ object CSVHook {
import p._
format.header.option.toSeq.flatMap(_.validate(inputs))
}
override def extension: String = ".csv"
}
}
......
......@@ -10,6 +10,7 @@ import org.openmole.core.workflow.builder._
trait OutputFormat[T] {
def write(format: T, output: WritableOutput, ps: Seq[Val[_]]): FromContext[Unit]
def validate(format: T): FromContextHook.ValidateParameters Seq[Throwable]
def extension: String
}
object FormattedFileHook {
......
......@@ -34,7 +34,7 @@ object AppendToFileHook {
Hook("AppendToFileHook") { p
import p._
val f = file.from(context)
f.createParentDir
f.createParentDirectory
// FIXME lock may not be necessary anymore - see you in 2020
f.withLock(_.append(content.from(context)))
context
......
......@@ -87,7 +87,7 @@ object CopyFileHook {
val from = context(filePrototype)
val to = destination.from(context)
to.createParentDir
to.createParentDirectory
val ret: Option[Variable[File]] =
if (options.move) {
from.realFile.move(to)
......
......@@ -65,7 +65,7 @@ object MatrixHook {
import parameters._
val f = file.from(context)
f.createParentDir
f.createParentDirectory
f.content = ""
for {
......
......@@ -62,7 +62,7 @@ object SaveHook {
import parameters._
val saveContext: Context = prototypes.map(p context.variable(p).getOrElse(throw new UserBadDataError(s"Variable $p has not been found")))
val to = file.from(context)
to.createParentDir
to.createParentDirectory
serializerService.serializeAndArchiveFiles(saveContext, to)
context
}
......
......@@ -27,6 +27,7 @@ object JSONOutputFormat {
}
override def validate(format: JSONOutputFormat): FromContextHook.ValidateParameters Seq[Throwable] = { p Seq() }
override def extension = ".json"
}
}
......
......@@ -46,7 +46,7 @@ object ABCHook {
thetai.map { _.formatted("%.12f") }.mkString(",")
}.mkString("\n")
file.createParentDir
file.createParentDirectory
file.content = header ++ "\n" ++ data
......
......@@ -28,7 +28,7 @@ object SavePopulationHook {
t.operations.result(context(t.populationPrototype).toVector, context(t.statePrototype)).from(context)
}
def hook(t: EvolutionWorkflow, output: WritableOutput, frequency: OptionalArgument[Long])(implicit name: sourcecode.Name, definitionScope: DefinitionScope) = {
def hook[F](t: EvolutionWorkflow, output: WritableOutput, frequency: OptionalArgument[Long], format: F = CSVOutputFormat(overwrite = true))(implicit name: sourcecode.Name, definitionScope: DefinitionScope, outputFormat: OutputFormat[F]) = {
Hook("SavePopulationHook") { p
import p._
import org.openmole.core.csv
......@@ -42,12 +42,13 @@ object SavePopulationHook {
}
if (save) {
val values = resultVariables(t).from(context).map(_.value)
def headerLine = csv.header(resultVariables(t).from(context).map(_.prototype.array), values)
output match {
case WritableOutput.FileValue(dir)
(dir / ExpandedString("population${" + t.generationPrototype.name + "}.csv")).from(context).withPrintStream(create = true) { ps
(dir / ExpandedString("population${" + t.generationPrototype.name + "}.csv")).from(context).withPrintStream(overwrite = false, create = true) { ps
csv.writeVariablesToCSV(
ps,
Some(headerLine),
......@@ -65,7 +66,7 @@ object SavePopulationHook {
}
context
} set (inputs += (t.populationPrototype, t.statePrototype))
} validate { p outputFormat.validate(format)(p) } set (inputs += (t.populationPrototype, t.statePrototype))
}
......
......@@ -129,7 +129,7 @@ object External {
}
private def copyFile(f: DeployedFile, to: File) = {
to.createParentDir
to.createParentDirectory
if (f.link) to.createLinkTo(f.file.getCanonicalFile)
else {
......
......@@ -384,7 +384,7 @@ package file {
}
}
def createParentDir = wrapError {
def createParentDirectory = wrapError {
file.getCanonicalFile.getParentFileSafe.mkdirs
}
......@@ -425,10 +425,22 @@ package file {
def withOutputStream[T] = withClosable[OutputStream, T](bufferedOutputStream())(_)
def withPrintStream[T](append: Boolean = false, create: Boolean = false) = {
if (create) file.createParentDir
if (create) file.createParentDirectory
withClosable[PrintStream, T](new PrintStream(file.bufferedOutputStream(append = append))) _
}
def atomicWithPrintStream[T](f: PrintStream T) = {
file.createParentDirectory
val tmpFile = java.io.File.createTempFile("printstream", ".tmp", file.getParentFile)
try {
val printStream = new PrintStream(file.bufferedOutputStream())
try f(printStream)
finally printStream.close()
}
finally Files.move(tmpFile.toPath, file.toPath, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING)
}
def withFileOutputStream[T] = withClosable[FileOutputStream, T](new FileOutputStream(file))(_)
def withInputStream[T] = withClosable[InputStream, T](bufferedInputStream)(_)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment