commit a32975e7725a7fe441c37a5f9b89d6b573451cd4 Author: Guillaume Martres Date: Tue Feb 19 20:44:23 2019 +0100 Add actorbintree assignment diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..349d2e8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,16 @@ +# General +*.DS_Store +*.swp +*~ + +# Dotty +*.class +*.tasty +*.hasTasty + +# sbt +target/ + +# Dotty IDE +/.dotty-ide-artifact +/.dotty-ide.json diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..c307fad --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,36 @@ +# DO NOT EDIT THIS FILE + +stages: + - build + - grade + +compile: + stage: build + image: lampepfl/moocs:dotty-2020-02-12 + except: + - tags + tags: + - cs206 + script: + - sbt packageSubmission + artifacts: + expire_in: 1 day + paths: + - submission.jar + +grade: + stage: grade + except: + - tags + tags: + - cs206 + image: + name: smarter3/moocs:reactive-actorbintree-2020-04-15 + entrypoint: [""] + allow_failure: true + before_script: + - mkdir -p /shared/submission/ + - cp submission.jar /shared/submission/submission.jar + script: + - cd /grader + - /grader/grade | /grader/feedback-printer diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..a35362b --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,8 @@ +{ + "dotty": { + "trace": { + "remoteTracingUrl": "wss://lamppc36.epfl.ch/dotty-remote-tracer/upload/lsp.log", + "server": { "format": "JSON", "verbosity": "verbose" } + } + } +} diff --git a/assignment.sbt b/assignment.sbt new file mode 100644 index 0000000..c6705a8 --- /dev/null +++ b/assignment.sbt @@ -0,0 +1,4 @@ +// Student tasks (i.e. submit, packageSubmission) +enablePlugins(StudentTasks) + + diff --git a/build.sbt b/build.sbt new file mode 100644 index 0000000..e08d686 --- /dev/null +++ b/build.sbt @@ -0,0 +1,24 @@ +course := "reactive" +assignment := "actorbintree" + +testOptions in Test += Tests.Argument(TestFrameworks.JUnit, "-a", "-v", "-s") +parallelExecution in Test := false + +val akkaVersion = "2.6.0" + +scalaVersion := "0.23.0-bin-20200211-5b006fb-NIGHTLY" + +scalacOptions ++= Seq( + "-feature", + "-deprecation", + "-encoding", "UTF-8", + "-unchecked", + "-language:implicitConversions" +) + +libraryDependencies ++= Seq( + "com.typesafe.akka" %% "akka-actor" % akkaVersion, + "com.typesafe.akka" %% "akka-testkit" % akkaVersion % Test, + "com.novocode" % "junit-interface" % "0.11" % Test +).map(_.withDottyCompat(scalaVersion.value)) +testSuite := "actorbintree.BinaryTreeSuite" diff --git a/grading-tests.jar b/grading-tests.jar new file mode 100644 index 0000000..3f08ec4 Binary files /dev/null and b/grading-tests.jar differ diff --git a/project/MOOCSettings.scala b/project/MOOCSettings.scala new file mode 100644 index 0000000..171244f --- /dev/null +++ b/project/MOOCSettings.scala @@ -0,0 +1,46 @@ +package ch.epfl.lamp + +import sbt._ +import sbt.Keys._ + +/** + * Coursera uses two versions of each assignment. They both have the same assignment key and part id but have + * different item ids. + * + * @param key Assignment key + * @param partId Assignment partId + * @param itemId Item id of the non premium version + * @param premiumItemId Item id of the premium version (`None` if the assignment is optional) + */ +case class CourseraId(key: String, partId: String, itemId: String, premiumItemId: Option[String]) + +/** + * Settings shared by all assignments, reused in various tasks. + */ +object MOOCSettings extends AutoPlugin { + + object autoImport { + val course = SettingKey[String]("course") + val assignment = SettingKey[String]("assignment") + val options = SettingKey[Map[String, Map[String, String]]]("options") + val courseraId = settingKey[CourseraId]("Coursera-specific information identifying the assignment") + val testSuite = settingKey[String]("Fully qualified name of the test suite of this assignment") + // Convenient alias + type CourseraId = ch.epfl.lamp.CourseraId + val CourseraId = ch.epfl.lamp.CourseraId + } + + import autoImport._ + + override val globalSettings: Seq[Def.Setting[_]] = Seq( + // supershell is verbose, buggy and useless. + useSuperShell := false + ) + + override val projectSettings: Seq[Def.Setting[_]] = Seq( + parallelExecution in Test := false, + // Report test result after each test instead of waiting for every test to finish + logBuffered in Test := false, + name := s"${course.value}-${assignment.value}" + ) +} diff --git a/project/StudentTasks.scala b/project/StudentTasks.scala new file mode 100644 index 0000000..7604830 --- /dev/null +++ b/project/StudentTasks.scala @@ -0,0 +1,318 @@ +package ch.epfl.lamp + +import sbt._ +import Keys._ + +// import scalaj.http._ +import java.io.{File, FileInputStream, IOException} +import org.apache.commons.codec.binary.Base64 +// import play.api.libs.json.{Json, JsObject, JsPath} +import scala.util.{Failure, Success, Try} + +/** + * Provides tasks for submitting the assignment + */ +object StudentTasks extends AutoPlugin { + + override def requires = super.requires && MOOCSettings + + object autoImport { + val packageSourcesOnly = TaskKey[File]("packageSourcesOnly", "Package the sources of the project") + val packageBinWithoutResources = TaskKey[File]("packageBinWithoutResources", "Like packageBin, but without the resources") + val packageSubmissionZip = TaskKey[File]("packageSubmissionZip") + val packageSubmission = inputKey[Unit]("package solution as an archive file") + val runGradingTests = taskKey[Unit]("run black-box tests used for final grading") + } + + + import autoImport._ + import MOOCSettings.autoImport._ + + override lazy val projectSettings = Seq( + packageSubmissionSetting, + // submitSetting, + runGradingTestsSettings, + + fork := true, + connectInput in run := true, + outputStrategy := Some(StdoutOutput), + ) ++ packageSubmissionZipSettings + + lazy val runGradingTestsSettings = runGradingTests := { + val testSuiteJar = "grading-tests.jar" + if (!new File(testSuiteJar).exists) { + throw new MessageOnlyException(s"Could not find tests JarFile: $testSuiteJar") + } + + val classPath = s"${(Test / dependencyClasspath).value.map(_.data).mkString(File.pathSeparator)}${File.pathSeparator}$testSuiteJar" + val junitProcess = + Fork.java.fork( + ForkOptions(), + "-cp" :: classPath :: + "org.junit.runner.JUnitCore" :: + (Test / testSuite).value :: + Nil + ) + + // Wait for tests to complete. + junitProcess.exitValue() + } + + + /** ********************************************************** + * SUBMITTING A SOLUTION TO COURSERA + */ + + val packageSubmissionZipSettings = Seq( + packageSubmissionZip := { + val submission = crossTarget.value / "submission.zip" + val sources = (packageSourcesOnly in Compile).value + val binaries = (packageBinWithoutResources in Compile).value + IO.zip(Seq(sources -> "sources.zip", binaries -> "binaries.jar"), submission) + submission + }, + artifactClassifier in packageSourcesOnly := Some("sources"), + artifact in (Compile, packageBinWithoutResources) ~= (art => art.withName(art.name + "-without-resources")) + ) ++ + inConfig(Compile)( + Defaults.packageTaskSettings(packageSourcesOnly, Defaults.sourceMappings) ++ + Defaults.packageTaskSettings(packageBinWithoutResources, Def.task { + val relativePaths = + (unmanagedResources in Compile).value.flatMap(Path.relativeTo((unmanagedResourceDirectories in Compile).value)(_)) + (mappings in (Compile, packageBin)).value.filterNot { case (_, path) => relativePaths.contains(path) } + }) + ) + + val maxSubmitFileSize = { + val mb = 1024 * 1024 + 10 * mb + } + + /** Check that the jar exists, isn't empty, isn't crazy big, and can be read + * If so, encode jar as base64 so we can send it to Coursera + */ + def prepareJar(jar: File, s: TaskStreams): String = { + val errPrefix = "Error submitting assignment jar: " + val fileLength = jar.length() + if (!jar.exists()) { + s.log.error(errPrefix + "jar archive does not exist\n" + jar.getAbsolutePath) + failSubmit() + } else if (fileLength == 0L) { + s.log.error(errPrefix + "jar archive is empty\n" + jar.getAbsolutePath) + failSubmit() + } else if (fileLength > maxSubmitFileSize) { + s.log.error(errPrefix + "jar archive is too big. Allowed size: " + + maxSubmitFileSize + " bytes, found " + fileLength + " bytes.\n" + + jar.getAbsolutePath) + failSubmit() + } else { + val bytes = new Array[Byte](fileLength.toInt) + val sizeRead = try { + val is = new FileInputStream(jar) + val read = is.read(bytes) + is.close() + read + } catch { + case ex: IOException => + s.log.error(errPrefix + "failed to read sources jar archive\n" + ex.toString) + failSubmit() + } + if (sizeRead != bytes.length) { + s.log.error(errPrefix + "failed to read the sources jar archive, size read: " + sizeRead) + failSubmit() + } else encodeBase64(bytes) + } + } + + /** Task to package solution to a given file path */ + lazy val packageSubmissionSetting = packageSubmission := { + val args: Seq[String] = Def.spaceDelimited("[path]").parsed + val s: TaskStreams = streams.value // for logging + val jar = (packageSubmissionZip in Compile).value + + val base64Jar = prepareJar(jar, s) + + val path = args.headOption.getOrElse((baseDirectory.value / "submission.jar").absolutePath) + scala.tools.nsc.io.File(path).writeAll(base64Jar) + } + +/* + /** Task to submit a solution to coursera */ + val submit = inputKey[Unit]("submit solution to Coursera") + lazy val submitSetting = submit := { + // Fail if scalafix linting does not pass. + scalafixLinting.value + + val args: Seq[String] = Def.spaceDelimited("").parsed + val s: TaskStreams = streams.value // for logging + val jar = (packageSubmissionZip in Compile).value + + val assignmentDetails = + courseraId.?.value.getOrElse(throw new MessageOnlyException("This assignment can not be submitted to Coursera because the `courseraId` setting is undefined")) + val assignmentKey = assignmentDetails.key + val courseName = + course.value match { + case "capstone" => "scala-capstone" + case "bigdata" => "scala-spark-big-data" + case other => other + } + + val partId = assignmentDetails.partId + val itemId = assignmentDetails.itemId + val premiumItemId = assignmentDetails.premiumItemId + + val (email, secret) = args match { + case email :: secret :: Nil => + (email, secret) + case _ => + val inputErr = + s"""|Invalid input to `submit`. The required syntax for `submit` is: + |submit + | + |The submit token is NOT YOUR LOGIN PASSWORD. + |It can be obtained from the assignment page: + |https://www.coursera.org/learn/$courseName/programming/$itemId + |${ + premiumItemId.fold("") { id => + s"""or (for premium learners): + |https://www.coursera.org/learn/$courseName/programming/$id + """.stripMargin + } + } + """.stripMargin + s.log.error(inputErr) + failSubmit() + } + + val base64Jar = prepareJar(jar, s) + val json = + s"""|{ + | "assignmentKey":"$assignmentKey", + | "submitterEmail":"$email", + | "secret":"$secret", + | "parts":{ + | "$partId":{ + | "output":"$base64Jar" + | } + | } + |}""".stripMargin + + def postSubmission[T](data: String): Try[HttpResponse[String]] = { + val http = Http("https://www.coursera.org/api/onDemandProgrammingScriptSubmissions.v1") + val hs = List( + ("Cache-Control", "no-cache"), + ("Content-Type", "application/json") + ) + s.log.info("Connecting to Coursera...") + val response = Try(http.postData(data) + .headers(hs) + .option(HttpOptions.connTimeout(10000)) // scalaj default timeout is only 100ms, changing that to 10s + .asString) // kick off HTTP POST + response + } + + val connectMsg = + s"""|Attempting to submit "${assignment.value}" assignment in "$courseName" course + |Using: + |- email: $email + |- submit token: $secret""".stripMargin + s.log.info(connectMsg) + + def reportCourseraResponse(response: HttpResponse[String]): Unit = { + val code = response.code + val respBody = response.body + + /* Sample JSON response from Coursera + { + "message": "Invalid email or token.", + "details": { + "learnerMessage": "Invalid email or token." + } + } + */ + + // Success, Coursera responds with 2xx HTTP status code + if (response.is2xx) { + val successfulSubmitMsg = + s"""|Successfully connected to Coursera. (Status $code) + | + |Assignment submitted successfully! + | + |You can see how you scored by going to: + |https://www.coursera.org/learn/$courseName/programming/$itemId/ + |${ + premiumItemId.fold("") { id => + s"""or (for premium learners): + |https://www.coursera.org/learn/$courseName/programming/$id + """.stripMargin + } + } + |and clicking on "My Submission".""".stripMargin + s.log.info(successfulSubmitMsg) + } + + // Failure, Coursera responds with 4xx HTTP status code (client-side failure) + else if (response.is4xx) { + val result = Try(Json.parse(respBody)).toOption + val learnerMsg = result match { + case Some(resp: JsObject) => + (JsPath \ "details" \ "learnerMessage").read[String].reads(resp).get + case Some(x) => // shouldn't happen + "Could not parse Coursera's response:\n" + x + case None => + "Could not parse Coursera's response:\n" + respBody + } + val failedSubmitMsg = + s"""|Submission failed. + |There was something wrong while attempting to submit. + |Coursera says: + |$learnerMsg (Status $code)""".stripMargin + s.log.error(failedSubmitMsg) + } + + // Failure, Coursera responds with 5xx HTTP status code (server-side failure) + else if (response.is5xx) { + val failedSubmitMsg = + s"""|Submission failed. + |Coursera seems to be unavailable at the moment (Status $code) + |Check https://status.coursera.org/ and try again in a few minutes. + """.stripMargin + s.log.error(failedSubmitMsg) + } + + // Failure, Coursera repsonds with an unexpected status code + else { + val failedSubmitMsg = + s"""|Submission failed. + |Coursera replied with an unexpected code (Status $code) + """.stripMargin + s.log.error(failedSubmitMsg) + } + } + + // kick it all off, actually make request + postSubmission(json) match { + case Success(resp) => reportCourseraResponse(resp) + case Failure(e) => + val failedConnectMsg = + s"""|Connection to Coursera failed. + |There was something wrong while attempting to connect to Coursera. + |Check your internet connection. + |${e.toString}""".stripMargin + s.log.error(failedConnectMsg) + } + + } +*/ + + def failSubmit(): Nothing = { + sys.error("Submission failed") + } + + /** + * ***************** + * DEALING WITH JARS + */ + def encodeBase64(bytes: Array[Byte]): String = + new String(Base64.encodeBase64(bytes)) +} diff --git a/project/build.properties b/project/build.properties new file mode 100644 index 0000000..a919a9b --- /dev/null +++ b/project/build.properties @@ -0,0 +1 @@ +sbt.version=1.3.8 diff --git a/project/buildSettings.sbt b/project/buildSettings.sbt new file mode 100644 index 0000000..8fac702 --- /dev/null +++ b/project/buildSettings.sbt @@ -0,0 +1,5 @@ +// Used for Coursera submission (StudentPlugin) +// libraryDependencies += "org.scalaj" %% "scalaj-http" % "2.4.2" +// libraryDependencies += "com.typesafe.play" %% "play-json" % "2.7.4" +// Used for Base64 (StudentPlugin) +libraryDependencies += "commons-codec" % "commons-codec" % "1.10" diff --git a/project/plugins.sbt b/project/plugins.sbt new file mode 100644 index 0000000..017735d --- /dev/null +++ b/project/plugins.sbt @@ -0,0 +1,2 @@ +addSbtPlugin("org.scala-js" % "sbt-scalajs" % "0.6.28") +addSbtPlugin("ch.epfl.lamp" % "sbt-dotty" % "0.4.0") diff --git a/src/main/scala/actorbintree/BinaryTreeSet.scala b/src/main/scala/actorbintree/BinaryTreeSet.scala new file mode 100644 index 0000000..e1c210d --- /dev/null +++ b/src/main/scala/actorbintree/BinaryTreeSet.scala @@ -0,0 +1,118 @@ +/** + * Copyright (C) 2009-2013 Typesafe Inc. + */ +package actorbintree + +import akka.actor._ +import scala.collection.immutable.Queue + +object BinaryTreeSet { + + trait Operation { + def requester: ActorRef + def id: Int + def elem: Int + } + + trait OperationReply { + def id: Int + } + + /** Request with identifier `id` to insert an element `elem` into the tree. + * The actor at reference `requester` should be notified when this operation + * is completed. + */ + case class Insert(requester: ActorRef, id: Int, elem: Int) extends Operation + + /** Request with identifier `id` to check whether an element `elem` is present + * in the tree. The actor at reference `requester` should be notified when + * this operation is completed. + */ + case class Contains(requester: ActorRef, id: Int, elem: Int) extends Operation + + /** Request with identifier `id` to remove the element `elem` from the tree. + * The actor at reference `requester` should be notified when this operation + * is completed. + */ + case class Remove(requester: ActorRef, id: Int, elem: Int) extends Operation + + /** Request to perform garbage collection */ + case object GC + + /** Holds the answer to the Contains request with identifier `id`. + * `result` is true if and only if the element is present in the tree. + */ + case class ContainsResult(id: Int, result: Boolean) extends OperationReply + + /** Message to signal successful completion of an insert or remove operation. */ + case class OperationFinished(id: Int) extends OperationReply + +} + + +class BinaryTreeSet extends Actor { + import BinaryTreeSet._ + import BinaryTreeNode._ + + def createRoot: ActorRef = context.actorOf(BinaryTreeNode.props(0, initiallyRemoved = true)) + + var root = createRoot + + // optional (used to stash incoming operations during garbage collection) + var pendingQueue = Queue.empty[Operation] + + // optional + def receive = normal + + // optional + /** Accepts `Operation` and `GC` messages. */ + val normal: Receive = { case _ => ??? } + + // optional + /** Handles messages while garbage collection is performed. + * `newRoot` is the root of the new binary tree where we want to copy + * all non-removed elements into. + */ + def garbageCollecting(newRoot: ActorRef): Receive = ??? + +} + +object BinaryTreeNode { + trait Position + + case object Left extends Position + case object Right extends Position + + case class CopyTo(treeNode: ActorRef) + /** + * Acknowledges that a copy has been completed. This message should be sent + * from a node to its parent, when this node and all its children nodes have + * finished being copied. + */ + case object CopyFinished + + def props(elem: Int, initiallyRemoved: Boolean) = Props(classOf[BinaryTreeNode], elem, initiallyRemoved) +} + +class BinaryTreeNode(val elem: Int, initiallyRemoved: Boolean) extends Actor { + import BinaryTreeNode._ + import BinaryTreeSet._ + + var subtrees = Map[Position, ActorRef]() + var removed = initiallyRemoved + + // optional + def receive = normal + + // optional + /** Handles `Operation` messages and `CopyTo` requests. */ + val normal: Receive = { case _ => ??? } + + // optional + /** `expected` is the set of ActorRefs whose replies we are waiting for, + * `insertConfirmed` tracks whether the copy of this node to the new tree has been confirmed. + */ + def copying(expected: Set[ActorRef], insertConfirmed: Boolean): Receive = ??? + + +} diff --git a/src/test/scala/actorbintree/BinaryTreeSuite.scala b/src/test/scala/actorbintree/BinaryTreeSuite.scala new file mode 100644 index 0000000..46ecca4 --- /dev/null +++ b/src/test/scala/actorbintree/BinaryTreeSuite.scala @@ -0,0 +1,126 @@ +/** + * Copyright (C) 2009-2015 Typesafe Inc. + */ +package actorbintree + +import akka.actor.{ActorRef, ActorSystem, Props, actorRef2Scala, scala2ActorRef} +import akka.testkit.{ImplicitSender, TestKit, TestProbe} +import org.junit.Test +import org.junit.Assert._ + +import scala.util.Random +import scala.concurrent.duration._ + +class BinaryTreeSuite extends TestKit(ActorSystem("BinaryTreeSuite")) with ImplicitSender { + + import actorbintree.BinaryTreeSet._ + + def receiveN(requester: TestProbe, ops: Seq[Operation], expectedReplies: Seq[OperationReply]): Unit = + requester.within(5.seconds) { + val repliesUnsorted = for (i <- 1 to ops.size) yield try { + requester.expectMsgType[OperationReply] + } catch { + case ex: Throwable if ops.size > 10 => sys.error(s"failure to receive confirmation $i/${ops.size}\n$ex") + case ex: Throwable => sys.error(s"failure to receive confirmation $i/${ops.size}\nRequests:" + ops.mkString("\n ", "\n ", "") + s"\n$ex") + } + val replies = repliesUnsorted.sortBy(_.id) + if (replies != expectedReplies) { + val pairs = (replies zip expectedReplies).zipWithIndex filter (x => x._1._1 != x._1._2) + fail("unexpected replies:" + pairs.map(x => s"at index ${x._2}: got ${x._1._1}, expected ${x._1._2}").mkString("\n ", "\n ", "")) + } + } + + def verify(probe: TestProbe, ops: Seq[Operation], expected: Seq[OperationReply]): Unit = { + val topNode = system.actorOf(Props[BinaryTreeSet]) + + ops foreach { op => + topNode ! op + } + + receiveN(probe, ops, expected) + // the grader also verifies that enough actors are created + } + + @Test def `proper inserts and lookups (5pts)`(): Unit = { + val topNode = system.actorOf(Props[BinaryTreeSet]) + + topNode ! Contains(testActor, id = 1, 1) + expectMsg(ContainsResult(1, false)) + + topNode ! Insert(testActor, id = 2, 1) + topNode ! Contains(testActor, id = 3, 1) + + expectMsg(OperationFinished(2)) + expectMsg(ContainsResult(3, true)) + () + } + + @Test def `instruction example (5pts)`(): Unit = { + val requester = TestProbe() + val requesterRef = requester.ref + val ops = List( + Insert(requesterRef, id=100, 1), + Contains(requesterRef, id=50, 2), + Remove(requesterRef, id=10, 1), + Insert(requesterRef, id=20, 2), + Contains(requesterRef, id=80, 1), + Contains(requesterRef, id=70, 2) + ) + + val expectedReplies = List( + OperationFinished(id=10), + OperationFinished(id=20), + ContainsResult(id=50, false), + ContainsResult(id=70, true), + ContainsResult(id=80, false), + OperationFinished(id=100) + ) + + verify(requester, ops, expectedReplies) + } + + + @Test def `behave identically to built-in set (includes GC) (40pts)`(): Unit = { + val rnd = new Random() + def randomOperations(requester: ActorRef, count: Int): Seq[Operation] = { + def randomElement: Int = rnd.nextInt(100) + def randomOperation(requester: ActorRef, id: Int): Operation = rnd.nextInt(4) match { + case 0 => Insert(requester, id, randomElement) + case 1 => Insert(requester, id, randomElement) + case 2 => Contains(requester, id, randomElement) + case 3 => Remove(requester, id, randomElement) + } + + for (seq <- 0 until count) yield randomOperation(requester, seq) + } + + def referenceReplies(operations: Seq[Operation]): Seq[OperationReply] = { + var referenceSet = Set.empty[Int] + def replyFor(op: Operation): OperationReply = op match { + case Insert(_, seq, elem) => + referenceSet = referenceSet + elem + OperationFinished(seq) + case Remove(_, seq, elem) => + referenceSet = referenceSet - elem + OperationFinished(seq) + case Contains(_, seq, elem) => + ContainsResult(seq, referenceSet(elem)) + } + + for (op <- operations) yield replyFor(op) + } + + val requester = TestProbe() + val topNode = system.actorOf(Props[BinaryTreeSet]) + val count = 1000 + + val ops = randomOperations(requester.ref, count) + val expectedReplies = referenceReplies(ops) + + ops foreach { op => + topNode ! op + if (rnd.nextDouble() < 0.1) topNode ! GC + } + receiveN(requester, ops, expectedReplies) + } +}