[Core] enh: start logging in file

openmole/bin/org.openmole.ui/src/main/scala/org/openmole/ui/Application.scala

@@ -21,7 +21,6 @@ import java.awt.Desktop
 import java.io.{ File, FileOutputStream, IOException }
 import java.util.logging.Level
 import java.net.URI
-
 import org.openmole.console.Console.ExitCodes
 import org.openmole.core.project._
 import org.openmole.core.compiler.ScalaREPL
@@ -72,6 +71,7 @@ object Application extends JavaLogger {
       ignored:                  List[String]    = Nil,
       port:                     Option[Int]     = None,
       loggerLevel:              Option[String]  = None,
+      loggerFileLevel:          Option[String]  = None,
       unoptimizedJS:            Boolean         = false,
       remote:                   Boolean         = false,
       http:                     Boolean         = false,
@@ -119,6 +119,7 @@ object Application extends JavaLogger {
       |[--reset-password] reset all preferences and ask for the a password
       |[--mem memory] allocate more memory to the JVM (not supported on windows yes), for instance --mem 2G
       |[--logger-level level] set the level of logging (OFF, SEVERE, WARNING, INFO, CONFIG, FINE, FINER, FINEST, ALL)
+      |[--logger-file-level level] set the level of logging if log file (OFF, SEVERE, WARNING, INFO, CONFIG, FINE, FINER, FINEST, ALL)
       |[--proxy hostname] set the proxy to use to install containers or R packages, in the form http://myproxy.org:3128
       |[--http-sub-directory] set the subdirectory for openmole app (for non-root path). No '/' is required (Example: "user1")
       |[--debug-no-output-redirection] deactivate system output redirection
@@ -146,6 +147,7 @@ object Application extends JavaLogger {
         case "--load-workspace-plugins" :: tail      ⇒ parse(tail, c.copy(loadHomePlugins = Some(true)))
         case "--console-work-directory" :: tail      ⇒ parse(dropArg(tail), c.copy(consoleWorkDirectory = Some(new File(takeArg(tail)))))
         case "--logger-level" :: tail                ⇒ parse(tail.tail, c.copy(loggerLevel = Some(tail.head)))
+        case "--logger-file-level" :: tail           ⇒ parse(tail.tail, c.copy(loggerFileLevel = Some(tail.head)))
         case "--remote" :: tail                      ⇒ parse(tail, c.copy(remote = true))
         case "--http" :: tail                        ⇒ parse(tail, c.copy(http = true))
         case "--no-browser" :: tail                  ⇒ parse(tail, c.copy(browse = false))
@@ -174,6 +176,7 @@ object Application extends JavaLogger {
 
     val logLevel = config.loggerLevel.map(l ⇒ Level.parse(l.toUpperCase))
     logLevel.foreach(LoggerConfig.level)
+    val logFileLevel = config.loggerFileLevel.map(l ⇒ Level.parse(l.toUpperCase))
 
     if (config.debugNoOutputRedirection) OutputManager.uninstall
 
@@ -233,7 +236,7 @@ object Application extends JavaLogger {
           Console.ExitCodes.incorrectPassword
         }
         else {
-          Services.withServices(workspaceDirectory, passwordString, config.proxyURI, logLevel) { services ⇒
+          Services.withServices(workspaceDirectory, passwordString, config.proxyURI, logLevel, logFileLevel) { services ⇒
             Runtime.getRuntime.addShutdownHook(thread(Services.dispose(services)))
             val server = new RESTServer(config.port, config.hostName, services, config.httpSubDirectory)
             server.run()
@@ -254,7 +257,7 @@ object Application extends JavaLogger {
         }
         else {
           Console.dealWithLoadError(loadPlugins, !config.scriptFile.isDefined)
-          Services.withServices(workspaceDirectory, passwordString, config.proxyURI, logLevel) { implicit services ⇒
+          Services.withServices(workspaceDirectory, passwordString, config.proxyURI, logLevel, logFileLevel) { implicit services ⇒
             Runtime.getRuntime.addShutdownHook(thread(Services.dispose(services)))
             val console = new Console(config.scriptFile)
             console.run(config.args, config.consoleWorkDirectory)
@@ -285,7 +288,7 @@ object Application extends JavaLogger {
 
             GUIServer.urlFile.content = url
 
-            GUIServerServices.withServices(workspace, config.proxyURI, logLevel) { services ⇒
+            GUIServerServices.withServices(workspace, config.proxyURI, logLevel, logFileLevel) { services ⇒
               Runtime.getRuntime.addShutdownHook(thread(GUIServerServices.dispose(services)))
               val server = new GUIServer(port, config.remote, useHTTP, services, config.password, extraHeader, !config.unoptimizedJS, config.httpSubDirectory)
               server.start()

openmole/build.sbt

@@ -409,7 +409,7 @@ def allEnvironment = Seq(batch, gridscale, ssh, egi, pbs, oar, sge, condor, slur
 lazy val batch = OsgiProject(pluginDir, "org.openmole.plugin.environment.batch", imports = Seq("*")) dependsOn(
   workflow, workspace, tools, event, replication, exception,
   serializer, fileService, pluginManager, openmoleTar, communication, authentication, location, services,
-  openmoleByteCode
+  openmoleByteCode, openmoleDSL
 ) settings (
   libraryDependencies ++= Seq(
     Libraries.gridscale,

openmole/core/org.openmole.core.dsl/src/main/scala/org/openmole/core/dsl/extension/package.scala

 package org.openmole.core.dsl
 
-import org.openmole.core.workflow.domain.{ DomainValidation, DomainInput }
+import org.openmole.core.workflow.domain.{ DomainInput, DomainValidation }
+import org.openmole.tool.logger.LoggerService
 
 package object extension {
 
@@ -107,4 +108,5 @@ package object extension {
   type Information = squants.information.Information
 
   type JavaLogger = org.openmole.tool.logger.JavaLogger
+  def Logger = LoggerService
 }

openmole/core/org.openmole.core.logconfig/src/main/scala/org/openmole/core/logconfig/LoggerConfig.scala

@@ -17,7 +17,7 @@
 
 package org.openmole.core.logconfig
 
-import org.apache.log4j.{ Logger ⇒ L4JLogger, Level ⇒ L4JLevel, Appender ⇒ L4JAppender }
+import org.apache.log4j.{ Appender ⇒ L4JAppender, Level ⇒ L4JLevel, Logger ⇒ L4JLogger }
 import org.apache.log4j.BasicConfigurator
 import java.util.logging._
 

openmole/core/org.openmole.core.services/src/main/scala/org/openmole/core/services/package.scala

@@ -37,8 +37,8 @@ package object services {
      * @tparam T
      * @return
      */
-    def withServices[T](workspace: File, password: String, httpProxy: Option[String], logLevel: Option[Level])(f: Services ⇒ T) = {
-      val services = Services(workspace, password, httpProxy, logLevel)
+    def withServices[T](workspace: File, password: String, httpProxy: Option[String], logLevel: Option[Level], logFileLevel: Option[Level])(f: Services ⇒ T) = {
+      val services = Services(workspace, password, httpProxy, logLevel, logFileLevel)
       try f(services)
       finally dispose(services)
     }
@@ -56,7 +56,12 @@ package object services {
      * @param httpProxy optional http proxy
      * @return
      */
-    def apply(workspace: File, password: String, httpProxy: Option[String], logLevel: Option[Level]) = {
+    def apply(
+      workspace:    File,
+      password:     String,
+      httpProxy:    Option[String],
+      logLevel:     Option[Level],
+      logFileLevel: Option[Level]) = {
       implicit val ws = Workspace(workspace)
       implicit val cypher = Cypher(password)
       implicit val preference = Services.preference(ws)
@@ -72,7 +77,7 @@ package object services {
       implicit val outputRedirection = OutputRedirection()
       implicit val networkService = NetworkService(httpProxy)
       implicit val fileServiceCache = FileServiceCache()
-      implicit val loggerService = LoggerService(logLevel)
+      implicit val loggerService = LoggerService(logLevel, file = Some(workspace / Workspace.logLocation), fileLevel = logFileLevel)
       implicit val timeService = TimeService()
 
       new ServicesContainer()

openmole/core/org.openmole.core.workflow/src/main/scala/org/openmole/core/workflow/mole/MoleExecution.scala

@@ -45,7 +45,7 @@ import org.openmole.tool.logger.{ JavaLogger, LoggerService }
 import scala.collection.mutable
 import scala.collection.mutable.{ Buffer, ListBuffer }
 
-object MoleExecution extends JavaLogger {
+object MoleExecution {
 
   class Started extends Event[MoleExecution]
   case class Finished(canceled: Boolean) extends Event[MoleExecution]
@@ -173,8 +173,8 @@ object MoleExecution extends JavaLogger {
             val ctx = try s.perform(subMoleExecutionState.moleExecution.implicits + context, subMoleExecutionState.moleExecution.executionContext)
             catch {
               case t: Throwable ⇒
-                Log.logger.log(Log.FINE, "Error in submole execution", t)
-                val event = MoleExecution.SourceExceptionRaised(s, capsule, t, Log.SEVERE)
+                LoggerService.fine("Error in submole execution", t)
+                val event = MoleExecution.SourceExceptionRaised(s, capsule, t, Level.SEVERE)
                 eventDispatcher.trigger(subMoleExecutionState.moleExecution, event)
                 cancel(subMoleExecutionState.moleExecution, Some(event))
                 throw new InternalProcessingError(t, s"Error in source execution that is plugged to $capsule")
@@ -276,7 +276,7 @@ object MoleExecution extends JavaLogger {
       catch {
         case e: Throwable ⇒
           import subMoleExecutionState.moleExecution.executionContext.services._
-          val event = MoleExecution.HookExceptionRaised(h, capsule, job, e, Log.SEVERE)
+          val event = MoleExecution.HookExceptionRaised(h, capsule, job, e, Level.SEVERE)
           eventDispatcher.trigger(subMoleExecutionState.moleExecution, event)
           cancel(subMoleExecutionState.moleExecution, Some(event))
           throw e
@@ -294,8 +294,9 @@ object MoleExecution extends JavaLogger {
     }
     catch {
       case t: Throwable ⇒
-        Log.logger.log(Log.FINE, "Error in submole execution", t)
-        val event = MoleExecution.ExceptionRaised(job, capsule, t, Log.SEVERE)
+        import subMoleExecutionState.moleExecution.executionContext.services._
+        LoggerService.fine("Error in submole execution", t)
+        val event = MoleExecution.ExceptionRaised(job, capsule, t, Level.SEVERE)
         import subMoleExecutionState.moleExecution.executionContext.services._
         eventDispatcher.trigger(subMoleExecutionState.moleExecution, event)
         cancel(subMoleExecutionState.moleExecution, Some(event))
@@ -320,9 +321,10 @@ object MoleExecution extends JavaLogger {
   def processFinalState(subMoleExecutionState: SubMoleExecutionState, job: JobId, result: Either[Context, Throwable], capsule: MoleCapsule, ticket: Ticket) = {
     result match {
       case Right(e) ⇒
+        import subMoleExecutionState.moleExecution.executionContext.services._
         val error = MoleExecution.JobFailed(job, capsule, e)
         cancel(subMoleExecutionState.moleExecution, Some(error))
-        Log.logger.log(Log.FINE, s"Error in user job execution for capsule $capsule, job state is FAILED.", e)
+        LoggerService.fine(s"Error in user job execution for capsule $capsule, job state is FAILED.", e)
         subMoleExecutionState.moleExecution.executionContext.services.eventDispatcher.trigger(subMoleExecutionState.moleExecution, error)
       case Left(context) ⇒
         subMoleExecutionState.moleExecution.completed(capsule) = subMoleExecutionState.moleExecution.completed(capsule) + 1
@@ -336,7 +338,7 @@ object MoleExecution extends JavaLogger {
   def start(moleExecution: MoleExecution, context: Option[Context]) =
     if (!moleExecution._started) {
       import moleExecution.executionContext.services._
-      LoggerService.log(Level.FINE, "Starting mole execution")
+      LoggerService.fine("Starting mole execution")
 
       def startEnvironments() = {
         if (moleExecution.startStopDefaultEnvironment) moleExecution.defaultEnvironment.start()
@@ -357,7 +359,7 @@ object MoleExecution extends JavaLogger {
   private def finish(moleExecution: MoleExecution, canceled: Boolean = false) =
     if (!moleExecution._finished) {
       import moleExecution.executionContext.services._
-      LoggerService.log(Level.FINE, s"finish mole execution $moleExecution, canceled ${canceled}")
+      LoggerService.fine(s"finish mole execution $moleExecution, canceled ${canceled}")
 
       moleExecution._finished = true
       moleExecution._endTime = Some(System.currentTimeMillis)

openmole/core/org.openmole.core.workspace/src/main/scala/org/openmole/core/workspace/Workspace.scala

@@ -21,7 +21,6 @@ import java.util.UUID
 import java.util.concurrent.atomic.AtomicLong
 import java.util.logging.Level
 import java.util.logging.Logger
-
 import org.jasypt.util.text._
 import org.openmole.core.event.{ Event, EventDispatcher }
 import org.openmole.core.exception.{ InternalProcessingError, UserBadDataError }
@@ -36,21 +35,21 @@ import org.openmole.tool.random.Random
 import org.openmole.tool.random.Random._
 
 import scala.concurrent.stm.{ Ref, atomic }
+import squants.information._
 
 object Workspace {
 
   case object PasswordRequired extends Event[Workspace]
 
-  def tmpLocation = ".tmp"
+  def tmpLocation = "tmp"
   def persistentLocation = "persistent"
+  def logLocation = "openmole.log.gz"
 
   def fixedPrefix = "file"
   def fixedPostfix = ".bin"
   def fixedDir = "dir"
 
-  lazy val defaultLocation = new File(System.getProperty("user.home"), s".openmole/${fixHostName}/")
-
-  // Workspace should be cleaned manualy
+  // Workspace should be cleaned after use
   def apply(location: File): Workspace = {
     val tmpDir = location / tmpLocation /> UUID.randomUUID.toString
     val persistentDir = location /> persistentLocation

openmole/gui/server/org.openmole.gui.server.core/src/main/scala/org/openmole/gui/server/core/GUIServlet.scala

@@ -81,7 +81,7 @@ object GUIServerServices {
     implicit def timeService: TimeService = guiServices.timeService
   }
 
-  def apply(workspace: Workspace, httpProxy: Option[String], logLevel: Option[Level]) = {
+  def apply(workspace: Workspace, httpProxy: Option[String], logLevel: Option[Level], logFileLevel: Option[Level]) = {
     implicit val ws = workspace
     implicit val preference = Preference(ws.persistentDir)
     implicit val newFile = TmpDirectory(workspace)
@@ -96,7 +96,7 @@ object GUIServerServices {
     implicit val networkService = NetworkService(httpProxy)
     implicit val fileServiceCache = FileServiceCache()
     implicit val replicaCatalog = ReplicaCatalog(ws)
-    implicit val loggerService = LoggerService(logLevel)
+    implicit val loggerService = LoggerService(logLevel, file = Some(workspace.location / Workspace.logLocation), fileLevel = logFileLevel)
     implicit val timeService = TimeService()
 
     new GUIServerServices()
@@ -107,8 +107,8 @@ object GUIServerServices {
     scala.util.Try(services.threadProvider.stop())
   }
 
-  def withServices[T](workspace: Workspace, httpProxy: Option[String], logLevel: Option[Level])(f: GUIServerServices ⇒ T) = {
-    val services = GUIServerServices(workspace, httpProxy, logLevel)
+  def withServices[T](workspace: Workspace, httpProxy: Option[String], logLevel: Option[Level], logFileLevel: Option[Level])(f: GUIServerServices ⇒ T) = {
+    val services = GUIServerServices(workspace, httpProxy, logLevel, logFileLevel)
     try f(services)
     finally dispose(services)
   }

openmole/plugins/org.openmole.plugin.environment.batch/src/main/scala/org/openmole/plugin/environment/batch/refresh/JobManager.scala

@@ -18,7 +18,6 @@
 package org.openmole.plugin.environment.batch.refresh
 
 import java.util.concurrent.TimeUnit
-
 import org.openmole.core.workflow.execution._
 import org.openmole.core.workflow.mole.MoleExecution.moleJobIsFinished
 import org.openmole.core.workflow.mole.{ MoleExecution, MoleExecutionMessage }
@@ -27,9 +26,11 @@ import org.openmole.plugin.environment.batch.environment.JobStore.StoredJob
 import org.openmole.plugin.environment.batch.environment._
 import org.openmole.tool.logger.JavaLogger
 import org.openmole.tool.thread._
+import org.openmole.core.dsl.extension.Logger
+
+import java.util.logging.Level
 
-object JobManager extends JavaLogger { self ⇒
-  import Log._
+object JobManager { self ⇒
 
   def killPriority = 10
 
@@ -90,11 +91,10 @@ object JobManager extends JavaLogger { self ⇒
              |$output""".stripMargin
         }.getOrElse(s"OpenMOLE output on remote node ${host} was empty")
 
-        val er = Environment.MoleJobExceptionRaised(j, e, WARNING, mj, detail = Some(detail))
+        val er = Environment.MoleJobExceptionRaised(j, e, Level.WARNING, mj, detail = Some(detail))
         environment.error(er)
         services.eventDispatcher.trigger(environment: Environment, er)
-        logger.log(FINE, "Error during job execution, it will be resubmitted.", e)
-
+        Logger.fine("Error during job execution, it will be resubmitted.", e)
     }
   }
 

openmole/plugins/org.openmole.plugin.environment.batch/src/main/scala/org/openmole/plugin/environment/batch/refresh/RefreshActor.scala

@@ -20,9 +20,10 @@ package org.openmole.plugin.environment.batch.refresh
 import org.openmole.core.exception.InternalProcessingError
 import org.openmole.core.workflow.execution.ExecutionState._
 import org.openmole.plugin.environment.batch.environment.{ BatchEnvironment, BatchJobControl }
-import org.openmole.tool.logger.JavaLogger
+import org.openmole.core.dsl._
+import org.openmole.core.dsl.extension._
 
-object RefreshActor extends JavaLogger {
+object RefreshActor {
 
   def receive(refresh: Refresh)(implicit services: BatchEnvironment.Services) = {
     import services._
@@ -57,7 +58,7 @@ object RefreshActor extends JavaLogger {
             JobManager ! Kill(job, environment, Some(bj))
           }
           else {
-            Log.logger.log(Log.FINE, s"${updateErrorsInARow + 1} errors in a row during job refresh", e)
+            Logger.fine(s"${updateErrorsInARow + 1} errors in a row during job refresh", e)
             JobManager ! Delay(Refresh(job, environment, bj, delay, updateErrorsInARow + 1), delay)
           }
       }

openmole/plugins/org.openmole.plugin.environment.batch/src/main/scala/org/openmole/plugin/environment/batch/refresh/RetryActionActor.scala

@@ -17,10 +17,10 @@
 
 package org.openmole.plugin.environment.batch.refresh
 
-import org.openmole.plugin.environment.batch.environment.{ BatchEnvironment, AccessControl }
-import org.openmole.tool.logger.JavaLogger
+import org.openmole.plugin.environment.batch.environment.{ AccessControl, BatchEnvironment }
+import org.openmole.tool.logger.{ JavaLogger, LoggerService }
 
-object RetryActionActor extends JavaLogger {
+object RetryActionActor {
 
   def receive(msg: RetryAction)(implicit services: BatchEnvironment.Services) = {
     import services._
@@ -32,7 +32,7 @@ object RetryActionActor extends JavaLogger {
     }
     catch {
       case t: Throwable ⇒
-        Log.logger.log(Log.FINE, "Error when deleting a file", t)
+        LoggerService.fine("Error when deleting a file", t)
     }
   }
 

openmole/project/Libraries.scala

@@ -87,7 +87,7 @@ object Libraries {
   lazy val spatialsampling = "org.openmole.library" %% "org-openmole-spatialsampling" % spatialsamplingVersion
   lazy val xzJava = "org.openmole.library" %% "xzjava" % "1.8"
   lazy val guava = "org.openmole.library" %% "com-google-guava" % guavaVersion
-  
+
   def httpClientVersion = "4.5.3"
   lazy val httpClient =
     Seq(

openmole/third-parties/org.openmole.tool.logger/src/main/scala/org/openmole/tool/logger/LoggerService.scala

@@ -17,24 +17,52 @@ package org.openmole.tool.logger
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-import java.util.logging.Level
-
 import org.openmole.tool.outputredirection._
 import sourcecode._
 
+import java.io.{ BufferedOutputStream, FileOutputStream, PrintStream }
+import java.util.logging.Level
+import java.util.zip.GZIPOutputStream
+
 object LoggerService {
-  def log(l: Level, msg: ⇒ String, exception: Option[Throwable] = None)(implicit name: FullName, line: Line, loggerService: LoggerService, outputRedirection: OutputRedirection) =
-    if (l.intValue() >= loggerService.level.getOrElse(Level.WARNING).intValue()) {
-      outputRedirection.error.println(s"""${name.value}:${line.value} - $msg""")
+
+  def log(l: Level, msg: ⇒ String, exception: Option[Throwable] = None)(implicit name: FullName, line: Line, loggerService: LoggerService, outputRedirection: OutputRedirection) = {
+    def outputError(p: PrintStream) = {
+      p.println(s"""${name.value}:${line.value} - $msg""")
       exception match {
         case Some(e) ⇒
-          outputRedirection.error.print(s"Caused by: ${e}")
-          e.printStackTrace(outputRedirection.error)
+          p.print(s"Caused by: ${e}")
+          e.printStackTrace(p)
         case None ⇒
       }
     }
 
-  def apply(level: Level): LoggerService = LoggerService(Some(level))
+    if (l.intValue() >= loggerService.level.getOrElse(Level.WARNING).intValue()) outputError(outputRedirection.error)
+
+    (loggerService.file, loggerService.fileLevel) match {
+      case (Some(f), Some(ll)) if l.intValue() >= ll.intValue() ⇒
+        f.getParentFile.mkdirs()
+        val ps = new PrintStream(new GZIPOutputStream(new BufferedOutputStream(new FileOutputStream(f, true))))
+        try outputError(ps)
+        finally ps.close()
+      case _ ⇒
+    }
+  }
+
+  def fine(msg: ⇒ String, exception: Throwable)(implicit name: FullName, line: Line, loggerService: LoggerService, outputRedirection: OutputRedirection) =
+    log(Level.FINE, msg, Some(exception))
+
+  def fine(msg: ⇒ String)(implicit name: FullName, line: Line, loggerService: LoggerService, outputRedirection: OutputRedirection) =
+    log(Level.FINE, msg, None)
+
+  def apply(
+    level:     Option[Level]        = None,
+    file:      Option[java.io.File] = None,
+    fileLevel: Option[Level]        = None) = new LoggerService(level, file, fileLevel)
+
 }
 
-case class LoggerService(level: Option[Level] = None)
+case class LoggerService(
+  level:     Option[Level],
+  file:      Option[java.io.File],
+  fileLevel: Option[Level])