Merge branch 'release/0.10.0'

.travis.yml

@@ -1,7 +1,7 @@
 language: scala
+sudo: false
 scala:
   - 2.10.4
-  - 2.9.3
 before_script:
   - mysql -u root -e "create database storehaus_test;"
   - mysql -u root -e "create user 'storehaususer'@'localhost' identified by 'test1234';"

CHANGES.md

@@ -1,5 +1,21 @@
 # Storehaus #
 
+### Version 0.10.0 ###
+* Use latest scalding, algebird, and bijection versions: https://github.com/twitter/storehaus/pull/255
+* Use new Travis CI container infrastructure: https://github.com/twitter/storehaus/pull/254
+* Add hook for CAS based memcache mergeable: https://github.com/twitter/storehaus/pull/252
+* Bump bijection/algebird versions: https://github.com/twitter/storehaus/pull/253
+* Remove + operator: https://github.com/twitter/storehaus/pull/21
+* Memcache mergeable - use semigroup: https://github.com/twitter/storehaus/pull/251
+* add logic for replicating writes and reads to stores: https://github.com/twitter/storehaus/pull/20
+* bump finagle and util to 6.22.0: https://github.com/twitter/storehaus/pull/247
+* Minified kill 2.9.3: https://github.com/twitter/storehaus/pull/249
+* Read through store - do not query backing store when no cache miss: https://github.com/twitter/storehaus/pull/246
+* implementation of store that uses http protocol: https://github.com/twitter/storehaus/pull/241
+* Retry unittest: https://github.com/twitter/storehaus/pull/240
+* Added endpoint support to storehaus-dynamodb: https://github.com/twitter/storehaus/pull/236
+* Https sonatype: https://github.com/twitter/storehaus/pull/237
+
 ### Version 0.9.1 ###
 * Feature/write through cache perf: https://github.com/twitter/storehaus/pull/234
 * Share the Retrying Read Write store in storehaus repo: https://github.com/twitter/storehaus/pull/230

README.md

@@ -98,6 +98,8 @@ Here's a list of modules we plan in implementing, with links to the github issue
 
 ## Community and Documentation
 
+This, and all [github.com/twitter](https://github.com/twitter) projects, are under the [Twitter Open Source Code of Conduct](https://engineering.twitter.com/opensource/code-of-conduct). Additionally, see the [Typelevel Code of Conduct](http://typelevel.org/conduct) for specific examples of harassing behavior that are not tolerated.
+
 To learn more and find links to tutorials and information around the web, check out the [Storehaus Wiki](https://github.com/twitter/storehaus/wiki).
 
 The latest ScalaDocs are hosted on Storehaus's [Github Project Page](http://twitter.github.io/storehaus).
@@ -106,7 +108,7 @@ Discussion occurs primarily on the [Storehaus mailing list](https://groups.googl
 
 ## Maven
 
-Storehaus modules are available on maven central. The current groupid and version for all modules is, respectively, `"com.twitter"` and  `0.9.0`.
+Storehaus modules are available on maven central. The current groupid and version for all modules is, respectively, `"com.twitter"` and  `0.10.0`.
 
 Current published artifacts are
 

project/Build.scala

@@ -57,9 +57,9 @@ object StorehausBuild extends Build {
 
   val sharedSettings = extraSettings ++ ciSettings ++ Seq(
     organization := "com.twitter",
-    scalaVersion := "2.9.3",
-    version := "0.9.1",
-    crossScalaVersions := Seq("2.9.3", "2.10.4"),
+    scalaVersion := "2.10.4",
+    version := "0.10.0",
+    crossScalaVersions := Seq("2.10.4"),
     javacOptions ++= Seq("-source", "1.6", "-target", "1.6"),
     javacOptions in doc := Seq("-source", "1.6"),
     libraryDependencies <+= scalaVersion(specs2Import(_)),
@@ -117,14 +117,14 @@ object StorehausBuild extends Build {
   def youngestForwardCompatible(subProj: String) =
     Some(subProj)
       .filterNot(unreleasedModules.contains(_))
-      .map { s => "com.twitter" % ("storehaus-" + s + "_2.9.3") % "0.9.0" }
-
-  val algebirdVersion = "0.7.0"
-  val bijectionVersion = "0.6.3"
-  val utilVersion = "6.11.0"
-  val scaldingVersion = "0.11.1"
+      .map { s => "com.twitter" % ("storehaus-" + s + "_2.10") % "0.10.0" }
 
+  val algebirdVersion = "0.9.0"
+  val bijectionVersion = "0.7.2"
+  val utilVersion = "6.22.0"
+  val scaldingVersion = "0.13.1"
   lazy val storehaus = Project(
+
     id = "storehaus",
     base = file("."),
     settings = sharedSettings ++ DocGen.publishSettings
@@ -145,6 +145,7 @@ object StorehausBuild extends Build {
     storehausKafka08,
     storehausMongoDB,
     storehausElastic,
+    storehausHttp,
     storehausTesting
   )
 
@@ -292,4 +293,12 @@ object StorehausBuild extends Build {
       javaOptions in run <++= (fullClasspath in Runtime) map { cp => Seq("-cp", sbt.Build.data(cp).mkString(":")) }
   ).dependsOn(storehausCore, storehausAlgebra, storehausCache)
 
+  lazy val storehausHttp = module("http").settings(
+    libraryDependencies ++= Seq(
+      Finagle.module("http"),
+      "com.twitter" %% "bijection-netty" % bijectionVersion
+    )
+  ).dependsOn(storehausCore)
+
+
 }

project/Finagle.scala

@@ -5,7 +5,7 @@ package storehaus
  *  dependency */
 object Finagle {
   import sbt._
-  val LatestVersion = "6.12.2"
+  val LatestVersion = "6.22.0"
   def module(name: String, version: String = LatestVersion) =
     StorehausBuild.withCross("com.twitter" %% "finagle-%s".format(name) % version)
 }

project/build.properties

@@ -1 +1 @@
-sbt.version=0.13.0
+sbt.version=0.13.5

project/plugins.sbt

@@ -1,6 +1,6 @@
 resolvers ++= Seq(
   "jgit-repo" at "http://download.eclipse.org/jgit/maven",
-  "sonatype-releases"  at "http://oss.sonatype.org/content/repositories/releases"
+  "sonatype-releases"  at "https://oss.sonatype.org/content/repositories/releases"
 )
 
 addSbtPlugin("com.typesafe.sbt" % "sbt-ghpages" % "0.5.1")
@@ -9,4 +9,4 @@ addSbtPlugin("com.typesafe" % "sbt-mima-plugin" % "0.1.6")
 
 addSbtPlugin("io.spray" % "sbt-boilerplate" % "0.5.1")
 
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2")
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2")

sbt

@@ -4,7 +4,7 @@
 # Author: Paul Phillips <[email protected]>
 
 # todo - make this dynamic
-declare -r sbt_release_version=0.13.0
+declare -r sbt_release_version=0.13.5
 
 declare sbt_jar sbt_dir sbt_create sbt_launch_dir
 declare scala_version java_home sbt_explicit_version

storehaus-cache/src/main/scala/com/twitter/storehaus/cache/CyclicIncrement.scala

@@ -59,7 +59,7 @@ class CyclicIncrementProvider[@specialized(Int, Long) K: Successible]
    nextSideCount: Int,
    maxNextSideVal: K) extends IdProvider[CyclicIncrement[K]] {
 
-  implicit val ord = implicitly[Successible[K]].ordering
+  implicit val ord = implicitly[Successible[K]].partialOrdering
   private def next(v: K) =
     Successible.next(v).getOrElse(throw new IllegalStateException("Hit maximum value for increment"))
 

storehaus-cache/src/main/scala/com/twitter/storehaus/cache/HHFilteredCache.scala

@@ -16,7 +16,7 @@
 
 package com.twitter.storehaus.cache
 
-import com.twitter.algebird.{ Semigroup, Monoid, Group, CMSHash  }
+import com.twitter.algebird.{ Semigroup, Monoid, Group, CMSHash, CMSHasherImplicits}
 import com.twitter.util.Future
 
 
@@ -41,6 +41,8 @@ object HeavyHittersPercent {
 }
 
 sealed class ApproxHHTracker[K](hhPct: HeavyHittersPercent, updateFreq: WriteOperationUpdateFrequency, roFreq: RollOverFrequencyMS) {
+  import CMSHasherImplicits._
+
   private[this] final val WIDTH = 1000
   private[this] final val DEPTH = 4
   private[this] final val hh = new java.util.HashMap[K, Long]()
@@ -53,9 +55,9 @@ sealed class ApproxHHTracker[K](hhPct: HeavyHittersPercent, updateFreq: WriteOpe
   private[this] var nextRollOver: Long = System.currentTimeMillis + roFreq.toLong
   private[this] final val updateOps = new java.util.concurrent.atomic.AtomicInteger(0)
 
-  private[this] final val hashes: IndexedSeq[CMSHash] = {
+  private[this] final val hashes: IndexedSeq[CMSHash[Long]] = {
     val r = new scala.util.Random(5)
-    (0 until DEPTH).map { _ => CMSHash(r.nextInt, 0, WIDTH) }
+    (0 until DEPTH).map { _ => CMSHash[Long](r.nextInt, 0, WIDTH) }
   }.toIndexedSeq
 
   @inline

storehaus-core/src/main/scala/com/twitter/storehaus/ReadThroughStore.scala

@@ -72,12 +72,17 @@ class ReadThroughStore[K, V](backingStore: ReadableStore[K, V], cache: Store[K,
         val hits = responses.filter { !_._2.isEmpty }
         val missedKeys = responses.filter { _._2.isEmpty }.keySet
 
-        FutureOps.mapCollect(backingStore.multiGet(missedKeys ++ failedKeys)).flatMap { storeResult =>
-          // write fetched keys to cache, best effort
-          mutex.acquire.flatMap { p =>
-            FutureOps.mapCollect(cache.multiPut(storeResult))(FutureCollector.bestEffort[(K1, Unit)])
-              .map { u => hits ++ storeResult }
-              .ensure { p.release }
+        val remaining = missedKeys ++ failedKeys
+        if (remaining.isEmpty) {
+          Future.value(hits) // no cache misses
+        } else {
+          FutureOps.mapCollect(backingStore.multiGet(remaining)).flatMap { storeResult =>
+            // write fetched keys to cache, best effort
+            mutex.acquire.flatMap { p =>
+              FutureOps.mapCollect(cache.multiPut(storeResult))(FutureCollector.bestEffort[(K1, Unit)])
+                .map { u => hits ++ storeResult }
+                .ensure { p.release }
+            }
           }
         }
       }

storehaus-core/src/test/scala/com/twitter/storehaus/RetryingStoreProperties.scala

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2014 Twitter Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License. You may obtain
+ * a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.twitter.storehaus
+
+import com.twitter.conversions.time._
+import com.twitter.util.{JavaTimer, Timer}
+
+import org.scalacheck.Properties
+
+object RetryingStoreProperties extends Properties("RetryingStore") {
+  import StoreProperties.storeTest
+
+  implicit val timer: Timer = new JavaTimer(true)
+
+  property("RetryingStore obeys the Store laws, assuming the underlying Store always returns results before timeout") =
+    storeTest[String, Int] {
+      Store.withRetry[String, Int](
+        store = new ConcurrentHashMapStore[String,Int](),
+        backoffs = for (i <- 0 until 3) yield 1.milliseconds
+      )(_ => true)
+    }
+}

storehaus-dynamodb/src/main/scala/com/twitter/storehaus/dynamodb/DynamoLongStore.scala

@@ -24,14 +24,18 @@ import com.twitter.storehaus.ConvertedStore
 import com.twitter.storehaus.algebra.MergeableStore
 
 import scala.util.Try
-
+import com.amazonaws.regions.{ Region, Regions }
 import com.amazonaws.services.dynamodbv2.model._
 
 import AwsBijections._
 
 object DynamoLongStore {
-  def apply(awsAccessKey: String, awsSecretKey: String, tableName: String, primaryKeyColumn: String, valueColumn: String) =
-    new DynamoLongStore(DynamoStore(awsAccessKey, awsSecretKey, tableName, primaryKeyColumn, valueColumn))
+  def apply(awsAccessKey: String, awsSecretKey: String, tableName: String,
+    primaryKeyColumn: String, valueColumn: String,
+    endpoint: Regions = Regions.US_EAST_1) =
+
+    new DynamoLongStore(DynamoStore(awsAccessKey, awsSecretKey, tableName,
+      primaryKeyColumn, valueColumn, endpoint))
 }
 
 class DynamoLongStore(underlying: DynamoStore)

storehaus-dynamodb/src/main/scala/com/twitter/storehaus/dynamodb/DynamoStore.scala

@@ -23,6 +23,7 @@ import com.twitter.util.{ Future, FuturePool }
 import com.twitter.storehaus.Store
 
 import com.amazonaws.auth.BasicAWSCredentials
+import com.amazonaws.regions.{ Region, Regions }
 import com.amazonaws.services.dynamodbv2.{ AmazonDynamoDBClient, AmazonDynamoDB }
 import com.amazonaws.services.dynamodbv2.model._
 
@@ -34,19 +35,25 @@ import AwsBijections._
 
 object DynamoStore {
 
-  def apply(awsAccessKey: String, awsSecretKey: String, tableName: String, primaryKeyColumn: String, valueColumn: String): DynamoStore = {
+  def apply(awsAccessKey: String, awsSecretKey: String, tableName: String,
+    primaryKeyColumn: String, valueColumn: String,
+    endpoint: Regions = Regions.US_EAST_1): DynamoStore = {
+
     val processors = Runtime.getRuntime.availableProcessors
-    this(awsAccessKey, awsSecretKey, tableName, primaryKeyColumn, valueColumn, processors)
+    this(awsAccessKey, awsSecretKey, tableName, primaryKeyColumn, valueColumn,
+      processors, endpoint)
   }
 
   def apply(awsAccessKey: String, awsSecretKey: String, tableName: String,
-    primaryKeyColumn: String, valueColumn: String, numberWorkerThreads: Int): DynamoStore = {
+    primaryKeyColumn: String, valueColumn: String, numberWorkerThreads: Int,
+    endpoint: Regions): DynamoStore = {
 
     val auth = new BasicAWSCredentials(awsAccessKey, awsSecretKey)
-    val client = new AmazonDynamoDBClient(auth)
-    new DynamoStore(client, tableName, primaryKeyColumn, valueColumn, numberWorkerThreads)
+    var client = new AmazonDynamoDBClient(auth)
+    client.setRegion(Region.getRegion(endpoint));
+    new DynamoStore(client, tableName, primaryKeyColumn, valueColumn,
+      numberWorkerThreads)
   }
-
 }
 
 class DynamoStore(val client: AmazonDynamoDB, val tableName: String,
@@ -75,9 +82,7 @@ class DynamoStore(val client: AmazonDynamoDB, val tableName: String,
 
         apiRequestFuturePool(client.deleteItem(deleteRequest))
       }
-
     }
-
   }
 
   override def get(k: String): Future[Option[AttributeValue]] = {
@@ -88,6 +93,4 @@ class DynamoStore(val client: AmazonDynamoDB, val tableName: String,
       Option(client.getItem(getRequest).getItem).map(_.get(valueColumn))
     }
   }
-
 }
-

storehaus-dynamodb/src/main/scala/com/twitter/storehaus/dynamodb/DynamoStringStore.scala

@@ -19,13 +19,18 @@ import java.util.{ Map => JMap }
 
 import com.twitter.storehaus.ConvertedStore
 
+import com.amazonaws.regions.{ Region, Regions }
 import com.amazonaws.services.dynamodbv2.model._
 
 import AwsBijections._
 
 object DynamoStringStore {
-  def apply(awsAccessKey: String, awsSecretKey: String, tableName: String, primaryKeyColumn: String, valueColumn: String) =
-    new DynamoStringStore(DynamoStore(awsAccessKey, awsSecretKey, tableName, primaryKeyColumn, valueColumn))
+  def apply(awsAccessKey: String, awsSecretKey: String, tableName: String,
+    primaryKeyColumn: String, valueColumn: String,
+    endpoint: Regions = Regions.US_EAST_1) =
+
+    new DynamoStringStore(DynamoStore(awsAccessKey, awsSecretKey, tableName,
+      primaryKeyColumn, valueColumn, endpoint))
 }
 
 class DynamoStringStore(underlying: DynamoStore)

storehaus-elasticsearch/src/test/scala/com/twitter/storehaus/elasticsearch/ElasticSearchStoreSpecs.scala

@@ -29,6 +29,8 @@ import org.json4s.{native, NoTypeHints}
  * @since 1/13/14
  */
 class ElasticSearchStoreSpecs extends Specification {
+  sequential
+
   private implicit val formats = native.Serialization.formats(NoTypeHints)
 
   private val person = Person("Joe", "Smith", 29)