commit 08fa05c1ec05f385d4bfdbf4b20bccc6a34b936d Author: Guillaume Martres Date: Tue Feb 19 20:44:23 2019 +0100 Add kmeans assignment diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..349d2e8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,16 @@ +# General +*.DS_Store +*.swp +*~ + +# Dotty +*.class +*.tasty +*.hasTasty + +# sbt +target/ + +# Dotty IDE +/.dotty-ide-artifact +/.dotty-ide.json diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..95220e2 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,36 @@ +# DO NOT EDIT THIS FILE + +stages: + - build + - grade + +compile: + stage: build + image: lampepfl/moocs:dotty-2020-02-12 + except: + - tags + tags: + - cs206 + script: + - sbt packageSubmission + artifacts: + expire_in: 1 day + paths: + - submission.jar + +grade: + stage: grade + except: + - tags + tags: + - cs206 + image: + name: smarter3/moocs:parprog1-kmeans-2020-02-24-2 + entrypoint: [""] + allow_failure: true + before_script: + - mkdir -p /shared/submission/ + - cp submission.jar /shared/submission/submission.jar + script: + - cd /grader + - /grader/grade | /grader/feedback-printer diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..a35362b --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,8 @@ +{ + "dotty": { + "trace": { + "remoteTracingUrl": "wss://lamppc36.epfl.ch/dotty-remote-tracer/upload/lsp.log", + "server": { "format": "JSON", "verbosity": "verbose" } + } + } +} diff --git a/assignment.sbt b/assignment.sbt new file mode 100644 index 0000000..79f63e6 --- /dev/null +++ b/assignment.sbt @@ -0,0 +1,9 @@ +// Student tasks (i.e. submit, packageSubmission) +enablePlugins(StudentTasks) + +courseraId := ch.epfl.lamp.CourseraId( + key = "UJmFEtoIEeWJwRKcpT8ChQ", + itemId = "Olt0g", + premiumItemId = Some("akLxD"), + partId = "mz8iL" +) diff --git a/build.sbt b/build.sbt new file mode 100644 index 0000000..584507b --- /dev/null +++ b/build.sbt @@ -0,0 +1,13 @@ +course := "parprog1" +assignment := "kmeans" + +scalaVersion := "0.23.0-bin-20200211-5b006fb-NIGHTLY" +scalacOptions ++= Seq("-language:implicitConversions", "-deprecation") +libraryDependencies ++= Seq( + "com.storm-enroute" %% "scalameter-core" % "0.19", + "org.scala-lang.modules" %% "scala-parallel-collections" % "0.2.0", + "com.novocode" % "junit-interface" % "0.11" % Test +).map(_.withDottyCompat(scalaVersion.value)) + +testOptions in Test += Tests.Argument(TestFrameworks.JUnit, "-a", "-v", "-s") +testSuite := "kmeans.KMeansSuite" diff --git a/grading-tests.jar b/grading-tests.jar new file mode 100644 index 0000000..be0f06c Binary files /dev/null and b/grading-tests.jar differ diff --git a/project/MOOCSettings.scala b/project/MOOCSettings.scala new file mode 100644 index 0000000..171244f --- /dev/null +++ b/project/MOOCSettings.scala @@ -0,0 +1,46 @@ +package ch.epfl.lamp + +import sbt._ +import sbt.Keys._ + +/** + * Coursera uses two versions of each assignment. They both have the same assignment key and part id but have + * different item ids. + * + * @param key Assignment key + * @param partId Assignment partId + * @param itemId Item id of the non premium version + * @param premiumItemId Item id of the premium version (`None` if the assignment is optional) + */ +case class CourseraId(key: String, partId: String, itemId: String, premiumItemId: Option[String]) + +/** + * Settings shared by all assignments, reused in various tasks. + */ +object MOOCSettings extends AutoPlugin { + + object autoImport { + val course = SettingKey[String]("course") + val assignment = SettingKey[String]("assignment") + val options = SettingKey[Map[String, Map[String, String]]]("options") + val courseraId = settingKey[CourseraId]("Coursera-specific information identifying the assignment") + val testSuite = settingKey[String]("Fully qualified name of the test suite of this assignment") + // Convenient alias + type CourseraId = ch.epfl.lamp.CourseraId + val CourseraId = ch.epfl.lamp.CourseraId + } + + import autoImport._ + + override val globalSettings: Seq[Def.Setting[_]] = Seq( + // supershell is verbose, buggy and useless. + useSuperShell := false + ) + + override val projectSettings: Seq[Def.Setting[_]] = Seq( + parallelExecution in Test := false, + // Report test result after each test instead of waiting for every test to finish + logBuffered in Test := false, + name := s"${course.value}-${assignment.value}" + ) +} diff --git a/project/StudentTasks.scala b/project/StudentTasks.scala new file mode 100644 index 0000000..7604830 --- /dev/null +++ b/project/StudentTasks.scala @@ -0,0 +1,318 @@ +package ch.epfl.lamp + +import sbt._ +import Keys._ + +// import scalaj.http._ +import java.io.{File, FileInputStream, IOException} +import org.apache.commons.codec.binary.Base64 +// import play.api.libs.json.{Json, JsObject, JsPath} +import scala.util.{Failure, Success, Try} + +/** + * Provides tasks for submitting the assignment + */ +object StudentTasks extends AutoPlugin { + + override def requires = super.requires && MOOCSettings + + object autoImport { + val packageSourcesOnly = TaskKey[File]("packageSourcesOnly", "Package the sources of the project") + val packageBinWithoutResources = TaskKey[File]("packageBinWithoutResources", "Like packageBin, but without the resources") + val packageSubmissionZip = TaskKey[File]("packageSubmissionZip") + val packageSubmission = inputKey[Unit]("package solution as an archive file") + val runGradingTests = taskKey[Unit]("run black-box tests used for final grading") + } + + + import autoImport._ + import MOOCSettings.autoImport._ + + override lazy val projectSettings = Seq( + packageSubmissionSetting, + // submitSetting, + runGradingTestsSettings, + + fork := true, + connectInput in run := true, + outputStrategy := Some(StdoutOutput), + ) ++ packageSubmissionZipSettings + + lazy val runGradingTestsSettings = runGradingTests := { + val testSuiteJar = "grading-tests.jar" + if (!new File(testSuiteJar).exists) { + throw new MessageOnlyException(s"Could not find tests JarFile: $testSuiteJar") + } + + val classPath = s"${(Test / dependencyClasspath).value.map(_.data).mkString(File.pathSeparator)}${File.pathSeparator}$testSuiteJar" + val junitProcess = + Fork.java.fork( + ForkOptions(), + "-cp" :: classPath :: + "org.junit.runner.JUnitCore" :: + (Test / testSuite).value :: + Nil + ) + + // Wait for tests to complete. + junitProcess.exitValue() + } + + + /** ********************************************************** + * SUBMITTING A SOLUTION TO COURSERA + */ + + val packageSubmissionZipSettings = Seq( + packageSubmissionZip := { + val submission = crossTarget.value / "submission.zip" + val sources = (packageSourcesOnly in Compile).value + val binaries = (packageBinWithoutResources in Compile).value + IO.zip(Seq(sources -> "sources.zip", binaries -> "binaries.jar"), submission) + submission + }, + artifactClassifier in packageSourcesOnly := Some("sources"), + artifact in (Compile, packageBinWithoutResources) ~= (art => art.withName(art.name + "-without-resources")) + ) ++ + inConfig(Compile)( + Defaults.packageTaskSettings(packageSourcesOnly, Defaults.sourceMappings) ++ + Defaults.packageTaskSettings(packageBinWithoutResources, Def.task { + val relativePaths = + (unmanagedResources in Compile).value.flatMap(Path.relativeTo((unmanagedResourceDirectories in Compile).value)(_)) + (mappings in (Compile, packageBin)).value.filterNot { case (_, path) => relativePaths.contains(path) } + }) + ) + + val maxSubmitFileSize = { + val mb = 1024 * 1024 + 10 * mb + } + + /** Check that the jar exists, isn't empty, isn't crazy big, and can be read + * If so, encode jar as base64 so we can send it to Coursera + */ + def prepareJar(jar: File, s: TaskStreams): String = { + val errPrefix = "Error submitting assignment jar: " + val fileLength = jar.length() + if (!jar.exists()) { + s.log.error(errPrefix + "jar archive does not exist\n" + jar.getAbsolutePath) + failSubmit() + } else if (fileLength == 0L) { + s.log.error(errPrefix + "jar archive is empty\n" + jar.getAbsolutePath) + failSubmit() + } else if (fileLength > maxSubmitFileSize) { + s.log.error(errPrefix + "jar archive is too big. Allowed size: " + + maxSubmitFileSize + " bytes, found " + fileLength + " bytes.\n" + + jar.getAbsolutePath) + failSubmit() + } else { + val bytes = new Array[Byte](fileLength.toInt) + val sizeRead = try { + val is = new FileInputStream(jar) + val read = is.read(bytes) + is.close() + read + } catch { + case ex: IOException => + s.log.error(errPrefix + "failed to read sources jar archive\n" + ex.toString) + failSubmit() + } + if (sizeRead != bytes.length) { + s.log.error(errPrefix + "failed to read the sources jar archive, size read: " + sizeRead) + failSubmit() + } else encodeBase64(bytes) + } + } + + /** Task to package solution to a given file path */ + lazy val packageSubmissionSetting = packageSubmission := { + val args: Seq[String] = Def.spaceDelimited("[path]").parsed + val s: TaskStreams = streams.value // for logging + val jar = (packageSubmissionZip in Compile).value + + val base64Jar = prepareJar(jar, s) + + val path = args.headOption.getOrElse((baseDirectory.value / "submission.jar").absolutePath) + scala.tools.nsc.io.File(path).writeAll(base64Jar) + } + +/* + /** Task to submit a solution to coursera */ + val submit = inputKey[Unit]("submit solution to Coursera") + lazy val submitSetting = submit := { + // Fail if scalafix linting does not pass. + scalafixLinting.value + + val args: Seq[String] = Def.spaceDelimited("").parsed + val s: TaskStreams = streams.value // for logging + val jar = (packageSubmissionZip in Compile).value + + val assignmentDetails = + courseraId.?.value.getOrElse(throw new MessageOnlyException("This assignment can not be submitted to Coursera because the `courseraId` setting is undefined")) + val assignmentKey = assignmentDetails.key + val courseName = + course.value match { + case "capstone" => "scala-capstone" + case "bigdata" => "scala-spark-big-data" + case other => other + } + + val partId = assignmentDetails.partId + val itemId = assignmentDetails.itemId + val premiumItemId = assignmentDetails.premiumItemId + + val (email, secret) = args match { + case email :: secret :: Nil => + (email, secret) + case _ => + val inputErr = + s"""|Invalid input to `submit`. The required syntax for `submit` is: + |submit + | + |The submit token is NOT YOUR LOGIN PASSWORD. + |It can be obtained from the assignment page: + |https://www.coursera.org/learn/$courseName/programming/$itemId + |${ + premiumItemId.fold("") { id => + s"""or (for premium learners): + |https://www.coursera.org/learn/$courseName/programming/$id + """.stripMargin + } + } + """.stripMargin + s.log.error(inputErr) + failSubmit() + } + + val base64Jar = prepareJar(jar, s) + val json = + s"""|{ + | "assignmentKey":"$assignmentKey", + | "submitterEmail":"$email", + | "secret":"$secret", + | "parts":{ + | "$partId":{ + | "output":"$base64Jar" + | } + | } + |}""".stripMargin + + def postSubmission[T](data: String): Try[HttpResponse[String]] = { + val http = Http("https://www.coursera.org/api/onDemandProgrammingScriptSubmissions.v1") + val hs = List( + ("Cache-Control", "no-cache"), + ("Content-Type", "application/json") + ) + s.log.info("Connecting to Coursera...") + val response = Try(http.postData(data) + .headers(hs) + .option(HttpOptions.connTimeout(10000)) // scalaj default timeout is only 100ms, changing that to 10s + .asString) // kick off HTTP POST + response + } + + val connectMsg = + s"""|Attempting to submit "${assignment.value}" assignment in "$courseName" course + |Using: + |- email: $email + |- submit token: $secret""".stripMargin + s.log.info(connectMsg) + + def reportCourseraResponse(response: HttpResponse[String]): Unit = { + val code = response.code + val respBody = response.body + + /* Sample JSON response from Coursera + { + "message": "Invalid email or token.", + "details": { + "learnerMessage": "Invalid email or token." + } + } + */ + + // Success, Coursera responds with 2xx HTTP status code + if (response.is2xx) { + val successfulSubmitMsg = + s"""|Successfully connected to Coursera. (Status $code) + | + |Assignment submitted successfully! + | + |You can see how you scored by going to: + |https://www.coursera.org/learn/$courseName/programming/$itemId/ + |${ + premiumItemId.fold("") { id => + s"""or (for premium learners): + |https://www.coursera.org/learn/$courseName/programming/$id + """.stripMargin + } + } + |and clicking on "My Submission".""".stripMargin + s.log.info(successfulSubmitMsg) + } + + // Failure, Coursera responds with 4xx HTTP status code (client-side failure) + else if (response.is4xx) { + val result = Try(Json.parse(respBody)).toOption + val learnerMsg = result match { + case Some(resp: JsObject) => + (JsPath \ "details" \ "learnerMessage").read[String].reads(resp).get + case Some(x) => // shouldn't happen + "Could not parse Coursera's response:\n" + x + case None => + "Could not parse Coursera's response:\n" + respBody + } + val failedSubmitMsg = + s"""|Submission failed. + |There was something wrong while attempting to submit. + |Coursera says: + |$learnerMsg (Status $code)""".stripMargin + s.log.error(failedSubmitMsg) + } + + // Failure, Coursera responds with 5xx HTTP status code (server-side failure) + else if (response.is5xx) { + val failedSubmitMsg = + s"""|Submission failed. + |Coursera seems to be unavailable at the moment (Status $code) + |Check https://status.coursera.org/ and try again in a few minutes. + """.stripMargin + s.log.error(failedSubmitMsg) + } + + // Failure, Coursera repsonds with an unexpected status code + else { + val failedSubmitMsg = + s"""|Submission failed. + |Coursera replied with an unexpected code (Status $code) + """.stripMargin + s.log.error(failedSubmitMsg) + } + } + + // kick it all off, actually make request + postSubmission(json) match { + case Success(resp) => reportCourseraResponse(resp) + case Failure(e) => + val failedConnectMsg = + s"""|Connection to Coursera failed. + |There was something wrong while attempting to connect to Coursera. + |Check your internet connection. + |${e.toString}""".stripMargin + s.log.error(failedConnectMsg) + } + + } +*/ + + def failSubmit(): Nothing = { + sys.error("Submission failed") + } + + /** + * ***************** + * DEALING WITH JARS + */ + def encodeBase64(bytes: Array[Byte]): String = + new String(Base64.encodeBase64(bytes)) +} diff --git a/project/build.properties b/project/build.properties new file mode 100644 index 0000000..a919a9b --- /dev/null +++ b/project/build.properties @@ -0,0 +1 @@ +sbt.version=1.3.8 diff --git a/project/buildSettings.sbt b/project/buildSettings.sbt new file mode 100644 index 0000000..8fac702 --- /dev/null +++ b/project/buildSettings.sbt @@ -0,0 +1,5 @@ +// Used for Coursera submission (StudentPlugin) +// libraryDependencies += "org.scalaj" %% "scalaj-http" % "2.4.2" +// libraryDependencies += "com.typesafe.play" %% "play-json" % "2.7.4" +// Used for Base64 (StudentPlugin) +libraryDependencies += "commons-codec" % "commons-codec" % "1.10" diff --git a/project/plugins.sbt b/project/plugins.sbt new file mode 100644 index 0000000..017735d --- /dev/null +++ b/project/plugins.sbt @@ -0,0 +1,2 @@ +addSbtPlugin("org.scala-js" % "sbt-scalajs" % "0.6.28") +addSbtPlugin("ch.epfl.lamp" % "sbt-dotty" % "0.4.0") diff --git a/src/main/resources/kmeans/epfl-view.jpg b/src/main/resources/kmeans/epfl-view.jpg new file mode 100644 index 0000000..5d66739 Binary files /dev/null and b/src/main/resources/kmeans/epfl-view.jpg differ diff --git a/src/main/scala/kmeans/KMeans.scala b/src/main/scala/kmeans/KMeans.scala new file mode 100644 index 0000000..ced54ea --- /dev/null +++ b/src/main/scala/kmeans/KMeans.scala @@ -0,0 +1,140 @@ +package kmeans + +import scala.annotation.tailrec +import scala.collection.{Map, Seq, mutable} +import scala.collection.parallel.CollectionConverters._ +import scala.collection.parallel.{ForkJoinTaskSupport, ParMap, ParSeq} +import scala.util.Random +import org.scalameter._ +import java.util.concurrent.ForkJoinPool + +class KMeans extends KMeansInterface { + + def generatePoints(k: Int, num: Int): ParSeq[Point] = { + val randx = new Random(1) + val randy = new Random(3) + val randz = new Random(5) + (0 until num) + .map({ i => + val x = ((i + 1) % k) * 1.0 / k + randx.nextDouble() * 0.5 + val y = ((i + 5) % k) * 1.0 / k + randy.nextDouble() * 0.5 + val z = ((i + 7) % k) * 1.0 / k + randz.nextDouble() * 0.5 + new Point(x, y, z) + }).to(mutable.ArrayBuffer).par + } + + def initializeMeans(k: Int, points: ParSeq[Point]): ParSeq[Point] = { + val rand = new Random(7) + (0 until k).map(_ => points(rand.nextInt(points.length))).to(mutable.ArrayBuffer).par + } + + def findClosest(p: Point, means: IterableOnce[Point]): Point = { + val it = means.iterator + assert(it.nonEmpty) + var closest = it.next() + var minDistance = p.squareDistance(closest) + while (it.hasNext) { + val point = it.next() + val distance = p.squareDistance(point) + if (distance < minDistance) { + minDistance = distance + closest = point + } + } + closest + } + + def classify(points: ParSeq[Point], means: ParSeq[Point]): ParMap[Point, ParSeq[Point]] = { + ??? + } + + def findAverage(oldMean: Point, points: ParSeq[Point]): Point = if (points.isEmpty) oldMean else { + var x = 0.0 + var y = 0.0 + var z = 0.0 + points.seq.foreach { p => + x += p.x + y += p.y + z += p.z + } + new Point(x / points.length, y / points.length, z / points.length) + } + + def update(classified: ParMap[Point, ParSeq[Point]], oldMeans: ParSeq[Point]): ParSeq[Point] = { + ??? + } + + def converged(eta: Double, oldMeans: ParSeq[Point], newMeans: ParSeq[Point]): Boolean = { + ??? + } + + @tailrec + final def kMeans(points: ParSeq[Point], means: ParSeq[Point], eta: Double): ParSeq[Point] = { + if (???) kMeans(???, ???, ???) else ??? // your implementation need to be tail recursive + } +} + +/** Describes one point in three-dimensional space. + * + * Note: deliberately uses reference equality. + */ +class Point(val x: Double, val y: Double, val z: Double) { + private def square(v: Double): Double = v * v + def squareDistance(that: Point): Double = { + square(that.x - x) + square(that.y - y) + square(that.z - z) + } + private def round(v: Double): Double = (v * 100).toInt / 100.0 + override def toString = s"(${round(x)}, ${round(y)}, ${round(z)})" +} + + +object KMeansRunner { + + val standardConfig = config( + Key.exec.minWarmupRuns -> 20, + Key.exec.maxWarmupRuns -> 40, + Key.exec.benchRuns -> 25, + Key.verbose -> false + ) withWarmer(new Warmer.Default) + + def main(args: Array[String]): Unit = { + val kMeans = new KMeans() + + val numPoints = 500000 + val eta = 0.01 + val k = 32 + val points = kMeans.generatePoints(k, numPoints) + val means = kMeans.initializeMeans(k, points) + + val seqtime = { + val parTasksupport = points.tasksupport + val seqPool = new ForkJoinPool(1) + val seqTasksupport = new ForkJoinTaskSupport(seqPool) + try { + points.tasksupport = seqTasksupport + means.tasksupport = seqTasksupport + standardConfig measure { + kMeans.kMeans(points, means, eta) + } + } + finally { + points.tasksupport = parTasksupport + means.tasksupport = parTasksupport + seqPool.shutdown() + } + } + + val partime = standardConfig measure { + kMeans.kMeans(points, means, eta) + } + + println(s"sequential time: $seqtime") + println(s"parallel time: $partime") + println(s"speedup: ${seqtime.value / partime.value}") + } + + // Workaround Dotty's handling of the existential type KeyValue + implicit def keyValueCoerce[T](kv: (Key[T], T)): KeyValue = { + kv.asInstanceOf[KeyValue] + } +} diff --git a/src/main/scala/kmeans/KMeansInterface.scala b/src/main/scala/kmeans/KMeansInterface.scala new file mode 100644 index 0000000..e74757d --- /dev/null +++ b/src/main/scala/kmeans/KMeansInterface.scala @@ -0,0 +1,15 @@ +package kmeans + +import scala.collection.{Map, Seq} +import scala.collection.parallel.{ParMap, ParSeq} + +/** + * The interface used by the grading infrastructure. Do not change signatures + * or your submission will fail with a NoSuchMethodError. + */ +trait KMeansInterface { + def classify(points: ParSeq[Point], means: ParSeq[Point]): ParMap[Point, ParSeq[Point]] + def update(classified: ParMap[Point, ParSeq[Point]], oldMeans: ParSeq[Point]): ParSeq[Point] + def converged(eta: Double, oldMeans: ParSeq[Point], newMeans: ParSeq[Point]): Boolean + def kMeans(points: ParSeq[Point], means: ParSeq[Point], eta: Double): ParSeq[Point] +} diff --git a/src/main/scala/kmeans/fun/IndexedColors.scala b/src/main/scala/kmeans/fun/IndexedColors.scala new file mode 100644 index 0000000..a0b1bcc --- /dev/null +++ b/src/main/scala/kmeans/fun/IndexedColors.scala @@ -0,0 +1,117 @@ +package kmeans +package fun + +import scala.collection.Seq +import scala.collection.parallel.ParSeq +import scala.collection.parallel.CollectionConverters._ + +abstract sealed trait InitialSelectionStrategy +case object RandomSampling extends InitialSelectionStrategy +case object UniformSampling extends InitialSelectionStrategy +case object UniformChoice extends InitialSelectionStrategy + +abstract sealed trait ConvergenceStrategy +case class ConvergedWhenSNRAbove(x: Double) extends ConvergenceStrategy +case class ConvergedAfterNSteps(n: Int) extends ConvergenceStrategy +case class ConvergedAfterMeansAreStill(eta: Double) extends ConvergenceStrategy + + +class IndexedColorFilter(initialImage: Img, + colorCount: Int, + initStrategy: InitialSelectionStrategy, + convStrategy: ConvergenceStrategy) extends KMeans { + + private var steps = 0 + + val points = imageToPoints(initialImage).par + val means = initializeIndex(colorCount, points).par + + /* The work is done here: */ + private val newMeans = kMeans(points, means, 0.01) + + /* And these are the results exposed */ + def getStatus() = s"Converged after $steps steps." + def getResult() = indexedImage(initialImage, newMeans) + + private def imageToPoints(img: Img): Seq[Point] = + for (x <- 0 until img.width; y <- 0 until img.height) yield { + val rgba = img(x, y) + new Point(red(rgba), green(rgba), blue(rgba)) + } + + private def indexedImage(img: Img, means: ParSeq[Point]) = { + val dst = new Img(img.width, img.height) + val pts = collection.mutable.Set[Point]() + + for (x <- 0 until img.width; y <- 0 until img.height) yield { + val v = img(x, y) + var point = new Point(red(v), green(v), blue(v)) + point = findClosest(point, means) + pts += point + dst(x, y) = rgba(point.x, point.y, point.z, 1d) + } + + dst + } + + private def initializeIndex(numColors: Int, points: ParSeq[Point]): Seq[Point] = { + val initialPoints: Seq[Point] = + initStrategy match { + case RandomSampling => + val d: Int = points.size / numColors + (0 until numColors) map (idx => points(d * idx)) + case UniformSampling => + val sep: Int = 32 + (for (r <- 0 until 255 by sep; g <- 0 until 255 by sep; b <- 0 until 255 by sep) yield { + def inside(p: Point): Boolean = + (p.x >= (r.toDouble / 255)) && + (p.x <= ((r.toDouble + sep) / 255)) && + (p.y >= (g.toDouble / 255)) && + (p.y <= ((g.toDouble + sep) / 255)) && + (p.z >= (b.toDouble / 255)) && + (p.z <= ((b.toDouble + sep) / 255)) + + val pts = points.filter(inside(_)) + val cnt = pts.size * 3 * numColors / points.size + if (cnt >= 1) { + val d = pts.size / cnt + (0 until cnt) map (idx => pts(d * idx)) + } else + Seq() + }).flatten + case UniformChoice => + val d: Int = math.max(1, (256 / math.cbrt(numColors.toDouble).ceil).toInt) + for (r <- 0 until 256 by d; g <- 0 until 256 by d; b <- 0 until 256 by d) yield + new Point(r.toDouble / 256,g.toDouble / 256, b.toDouble / 256) + } + + val d2 = initialPoints.size.toDouble / numColors + (0 until numColors) map (idx => initialPoints((idx * d2).toInt)) + } + + private def computeSNR(points: ParSeq[Point], means: ParSeq[Point]): Double = { + var sound = 0.0 + var noise = 0.0 + + for (point <- points) { + import math.{pow, sqrt} + val closest = findClosest(point, means) + sound += sqrt(pow(point.x, 2) + pow(point.y, 2) + pow(point.z, 2)) + noise += sqrt(pow(point.x - closest.x, 2) + pow(point.y - closest.y, 2) + pow(point.z - closest.z, 2)) + } + sound/noise + } + + override def converged(eta: Double, oldMeans: ParSeq[Point], newMeans: ParSeq[Point]): Boolean = { + steps += 1 + convStrategy match { + case ConvergedAfterNSteps(n) => + steps >= n + case ConvergedAfterMeansAreStill(eta) => + super.converged(eta, oldMeans, newMeans) + case ConvergedWhenSNRAbove(snr_desired) => + val snr_computed = computeSNR(points.par, newMeans) + snr_computed >= snr_desired + } + } +} diff --git a/src/main/scala/kmeans/fun/PhotoCanvas.scala b/src/main/scala/kmeans/fun/PhotoCanvas.scala new file mode 100644 index 0000000..2c09f3d --- /dev/null +++ b/src/main/scala/kmeans/fun/PhotoCanvas.scala @@ -0,0 +1,95 @@ +package kmeans +package fun + +import java.awt._ +import java.awt.event._ +import java.awt.image._ +import java.io._ +import javax.imageio._ +import javax.swing._ +import javax.swing.event._ + +class PhotoCanvas extends JComponent { + + var imagePath: Option[String] = None + + var image = loadEPFLImage() + + val timerDelay = 100 + val timer = + new Timer(timerDelay, new ActionListener() { + def actionPerformed(e: ActionEvent): Unit = repaint() + }) + + override def getPreferredSize = { + new Dimension(image.width, image.height) + } + + private def loadEPFLImage(): Img = { + val stream = this.getClass.getResourceAsStream("/kmeans/epfl-view.jpg") + try { + loadImage(stream) + } finally { + stream.close() + } + } + + private def loadFileImage(path: String): Img = { + val stream = new FileInputStream(path) + try { + loadImage(stream) + } finally { + stream.close() + } + } + + private def loadImage(inputStream: InputStream): Img = { + val bufferedImage = ImageIO.read(inputStream) + val width = bufferedImage.getWidth + val height = bufferedImage.getHeight + val img = new Img(width, height) + for (x <- 0 until width; y <- 0 until height) + img(x, y) = bufferedImage.getRGB(x, y) + img + } + + def reload(): Unit = { + image = imagePath match { + case Some(path) => loadFileImage(path) + case None => loadEPFLImage() + } + repaint() + } + + def loadFile(path: String): Unit = { + imagePath = Some(path) + reload() + } + + def saveFile(path: String): Unit = { + reload() + val stream = new FileOutputStream(path) + val bufferedImage = new BufferedImage(image.width, image.height, BufferedImage.TYPE_INT_ARGB) + for (x <- 0 until image.width; y <- 0 until image.height) bufferedImage.setRGB(x, y, image(x, y)) + ImageIO.write(bufferedImage, "png", stream) + } + + def applyIndexedColors(colorCount: Int, initStrategy: InitialSelectionStrategy, convStrategy: ConvergenceStrategy): String = { + val filter = new IndexedColorFilter(image, colorCount, initStrategy, convStrategy) + image = filter.getResult() + repaint() + filter.getStatus() + } + + override def paintComponent(gcan: Graphics) = { + super.paintComponent(gcan) + + val width = image.width + val height = image.height + val bufferedImage = new BufferedImage(width, height, BufferedImage.TYPE_INT_ARGB) + for (x <- 0 until width; y <- 0 until height) bufferedImage.setRGB(x, y, image(x, y)) + + gcan.drawImage(bufferedImage, 0, 0, null) + } + +} diff --git a/src/main/scala/kmeans/fun/ScalaShop.scala b/src/main/scala/kmeans/fun/ScalaShop.scala new file mode 100644 index 0000000..f8eb280 --- /dev/null +++ b/src/main/scala/kmeans/fun/ScalaShop.scala @@ -0,0 +1,218 @@ +package kmeans +package fun + +import java.awt._ +import java.awt.event._ +import javax.swing._ +import javax.swing.event._ +import scala.collection.mutable.ArrayBuffer +import scala.reflect.ClassTag +import org.scalameter._ + +object ScalaShop { + + class ScalaShopFrame extends JFrame("ScalaShop\u2122") { + setDefaultCloseOperation(WindowConstants.EXIT_ON_CLOSE) + setSize(800, 500) + setLayout(new BorderLayout) + + val rightpanel = new JPanel + rightpanel.setBorder(BorderFactory.createEtchedBorder(border.EtchedBorder.LOWERED)) + rightpanel.setLayout(new BorderLayout) + add(rightpanel, BorderLayout.EAST) + + val allControls = new JPanel + allControls.setLayout(new BoxLayout(allControls, BoxLayout.Y_AXIS)) + rightpanel.add(allControls, BorderLayout.NORTH) + + // Color count selection + val colorControls = new JPanel + colorControls.setLayout(new GridLayout(0, 2)) + allControls.add(colorControls) + + val colorCountLabel = new JLabel("Colors") + colorControls.add(colorCountLabel) + + val colorCountSpinner = new JSpinner(new SpinnerNumberModel(32, 16, 512, 16)) + colorControls.add(colorCountSpinner) + + // Initial selection + val initSelectionControls = new JPanel + initSelectionControls.setLayout(new GridLayout(0, 1)) + allControls.add(initSelectionControls) + + val initialSelectionGroup = new ButtonGroup() + + val initSelectionLabel = new JLabel("Initial Color Selection:") + initSelectionControls.add(initSelectionLabel) + + val uniformSamplingButton = new JRadioButton("Uniform Sampling") + uniformSamplingButton.setSelected(true); + initSelectionControls.add(uniformSamplingButton) + + val randomSamplingButton = new JRadioButton("Random Sampling") + initSelectionControls.add(randomSamplingButton) + + val uniformChoiceButton = new JRadioButton("Uniform Choice") + initSelectionControls.add(uniformChoiceButton) + + initialSelectionGroup.add(randomSamplingButton) + initialSelectionGroup.add(uniformSamplingButton) + initialSelectionGroup.add(uniformChoiceButton) + + // Initial means selection + val convergenceControls = new JPanel + convergenceControls.setLayout(new BoxLayout(convergenceControls, BoxLayout.Y_AXIS)) + allControls.add(convergenceControls) + + val convergenceGroup = new ButtonGroup() + + val convergenceLabel = new JLabel("Convergence criteria:") + initSelectionControls.add(convergenceLabel) + + val criteriaControls = new JPanel + criteriaControls.setLayout(new GridLayout(0, 2)) + convergenceControls.add(criteriaControls) + + val stepConvergenceButton = new JRadioButton("Steps") + criteriaControls.add(stepConvergenceButton) + + val stepCountSpinner = new JSpinner(new SpinnerNumberModel(5, 1, 50, 1)) + criteriaControls.add(stepCountSpinner) + + val etaConvergenceButton = new JRadioButton("Eta") + etaConvergenceButton.setSelected(true); + criteriaControls.add(etaConvergenceButton) + + val etaCountSpinner = new JSpinner(new SpinnerNumberModel(0.001, 0.00001, 0.01, 0.00001)) + criteriaControls.add(etaCountSpinner) + + val snrConvergenceButton = new JRadioButton("Sound-to-noise") + criteriaControls.add(snrConvergenceButton) + + val snrCountSpinner = new JSpinner(new SpinnerNumberModel(40, 10, 80, 1)) + criteriaControls.add(snrCountSpinner) + + convergenceGroup.add(stepConvergenceButton) + convergenceGroup.add(etaConvergenceButton) + convergenceGroup.add(snrConvergenceButton) + + // Action Buttons + val actionControls = new JPanel + actionControls.setLayout(new GridLayout(0, 2)) + allControls.add(actionControls) + + val stepbutton = new JButton("Apply filter") + stepbutton.addActionListener(new ActionListener { + def actionPerformed(e: ActionEvent): Unit = { + var status = "" + val time = measure { + status = canvas.applyIndexedColors(getColorCount, getInitialSelectionStrategy, getConvergenceStragegy) + } + updateInformationBox(status, time.value) + } + }) + actionControls.add(stepbutton) + + val clearButton = new JButton("Reload") + clearButton.addActionListener(new ActionListener { + def actionPerformed(e: ActionEvent): Unit = { + canvas.reload() + } + }) + actionControls.add(clearButton) + + val info = new JTextArea(" ") + info.setBorder(BorderFactory.createLoweredBevelBorder) + rightpanel.add(info, BorderLayout.SOUTH) + + val mainMenuBar = new JMenuBar() + + val fileMenu = new JMenu("File") + val openMenuItem = new JMenuItem("Open...") + openMenuItem.addActionListener(new ActionListener { + def actionPerformed(e: ActionEvent): Unit = { + val fc = new JFileChooser() + if (fc.showOpenDialog(ScalaShopFrame.this) == JFileChooser.APPROVE_OPTION) { + canvas.loadFile(fc.getSelectedFile.getPath) + } + } + }) + fileMenu.add(openMenuItem) + val saveMenuItem = new JMenuItem("Save...") + saveMenuItem.addActionListener(new ActionListener { + def actionPerformed(e: ActionEvent): Unit = { + val fc = new JFileChooser("epfl-view.png") + if (fc.showSaveDialog(ScalaShopFrame.this) == JFileChooser.APPROVE_OPTION) { + canvas.saveFile(fc.getSelectedFile.getPath) + } + } + }) + fileMenu.add(saveMenuItem) + val exitMenuItem = new JMenuItem("Exit") + exitMenuItem.addActionListener(new ActionListener { + def actionPerformed(e: ActionEvent): Unit = { + sys.exit(0) + } + }) + fileMenu.add(exitMenuItem) + + mainMenuBar.add(fileMenu) + + val helpMenu = new JMenu("Help") + val aboutMenuItem = new JMenuItem("About") + aboutMenuItem.addActionListener(new ActionListener { + def actionPerformed(e: ActionEvent): Unit = { + JOptionPane.showMessageDialog(null, "ScalaShop, the ultimate image manipulation tool\nBrought to you by EPFL, 2015") + } + }) + helpMenu.add(aboutMenuItem) + + mainMenuBar.add(helpMenu) + + setJMenuBar(mainMenuBar) + + val canvas = new PhotoCanvas + + val scrollPane = new JScrollPane(canvas) + + add(scrollPane, BorderLayout.CENTER) + setVisible(true) + + def updateInformationBox(status: String, time: Double): Unit = { + info.setText(s"$status\nTime: ${time.toInt} ms.") + } + + def getColorCount: Int = + colorCountSpinner.getValue.asInstanceOf[Int] + + def getInitialSelectionStrategy: InitialSelectionStrategy = + if (randomSamplingButton.isSelected()) + RandomSampling + else if (uniformSamplingButton.isSelected()) + UniformSampling + else + UniformChoice + + def getConvergenceStragegy: ConvergenceStrategy = + if (stepConvergenceButton.isSelected()) + ConvergedAfterNSteps(stepCountSpinner.getValue.asInstanceOf[Int]) + else if (etaConvergenceButton.isSelected()) + ConvergedAfterMeansAreStill(etaCountSpinner.getValue.asInstanceOf[Double]) + else + ConvergedWhenSNRAbove(snrCountSpinner.getValue.asInstanceOf[Int]) + } + + try { + UIManager.setLookAndFeel(UIManager.getSystemLookAndFeelClassName()) + } catch { + case _: Exception => println("Cannot set look and feel, using the default one.") + } + + val frame = new ScalaShopFrame + + def main(args: Array[String]): Unit = { + frame.repaint() + } + +} diff --git a/src/main/scala/kmeans/fun/package.scala b/src/main/scala/kmeans/fun/package.scala new file mode 100644 index 0000000..e4e1827 --- /dev/null +++ b/src/main/scala/kmeans/fun/package.scala @@ -0,0 +1,41 @@ +package kmeans + +package object fun { + + /** The value of every pixel is represented as a 32 bit integer. */ + type RGBA = Int + + /** Returns the alpha component. */ + def alpha(c: RGBA): Double = ((0xff000000 & c) >>> 24).toDouble / 256 + + /** Returns the red component. */ + def red(c: RGBA): Double = ((0x00ff0000 & c) >>> 16).toDouble / 256 + + /** Returns the green component. */ + def green(c: RGBA): Double = ((0x0000ff00 & c) >>> 8).toDouble / 256 + + /** Returns the blue component. */ + def blue(c: RGBA): Double = ((0x000000ff & c) >>> 0).toDouble / 256 + + /** Used to create an RGBA value from separate components. */ + def rgba(r: Double, g: Double, b: Double, a: Double): RGBA = { + (clamp((a * 256).toInt, 0, 255) << 24) | + (clamp((r * 256).toInt, 0, 255) << 16) | + (clamp((g * 256).toInt, 0, 255) << 8) | + (clamp((b * 256).toInt, 0, 255) << 0) + } + + /** Restricts the integer into the specified range. */ + def clamp(v: Int, min: Int, max: Int): Int = { + if (v < min) min + else if (v > max) max + else v + } + + /** Image is a two-dimensional matrix of pixel values. */ + class Img(val width: Int, val height: Int, private val data: Array[RGBA]) { + def this(w: Int, h: Int) = this(w, h, new Array(w * h)) + def apply(x: Int, y: Int): RGBA = data(y * width + x) + def update(x: Int, y: Int, c: RGBA): Unit = data(y * width + x) = c + } +} diff --git a/src/test/scala/kmeans/KMeansSuite.scala b/src/test/scala/kmeans/KMeansSuite.scala new file mode 100644 index 0000000..4f46c72 --- /dev/null +++ b/src/test/scala/kmeans/KMeansSuite.scala @@ -0,0 +1,28 @@ +package kmeans + +import java.util.concurrent._ +import scala.collection.{mutable, Map, Seq} +import scala.collection.parallel.{ParMap, ParSeq} +import scala.collection.parallel.CollectionConverters._ +import scala.math._ +import org.junit._ +import org.junit.Assert.assertEquals + +class KMeansSuite { + object KM extends KMeans + import KM._ + + def checkParClassify(points: ParSeq[Point], means: ParSeq[Point], expected: ParMap[Point, ParSeq[Point]]): Unit = { + assertEquals(s"classify($points, $means) should equal to $expected", expected, classify(points, means)) + } + + @Test def `'classify' should work for empty 'points' and empty 'means'`: Unit = { + val points: ParSeq[Point] = IndexedSeq().par + val means: ParSeq[Point] = IndexedSeq().par + val expected = ParMap[Point, ParSeq[Point]]() + checkParClassify(points, means, expected) + } + +} + +