diff --git a/src/main/scala/io/findify/s3mock/provider/FileProvider.scala b/src/main/scala/io/findify/s3mock/provider/FileProvider.scala index 99c67a2..120aa1d 100644 --- a/src/main/scala/io/findify/s3mock/provider/FileProvider.scala +++ b/src/main/scala/io/findify/s3mock/provider/FileProvider.scala @@ -1,14 +1,15 @@ package io.findify.s3mock.provider +import java.io.{FileInputStream, File ⇒ JFile} import java.util.UUID -import java.io.{FileInputStream, File => JFile} import akka.http.scaladsl.model.DateTime import better.files.File import better.files.File.OpenOptions -import com.amazonaws.services.s3.model.ObjectMetadata +import com.amazonaws.services.s3.model.{ObjectMetadata, Tag} import com.typesafe.scalalogging.LazyLogging import io.findify.s3mock.error.{NoSuchBucketException, NoSuchKeyException} import io.findify.s3mock.provider.metadata.{MapMetadataStore, MetadataStore} +import io.findify.s3mock.provider.tags.InMemoryTagStore import io.findify.s3mock.request.{CompleteMultipartUpload, CreateBucketConfiguration} import io.findify.s3mock.response._ import org.apache.commons.codec.digest.DigestUtils @@ -23,6 +24,7 @@ class FileProvider(dir:String) extends Provider with LazyLogging { if (!workDir.exists) workDir.createDirectories() private val meta = new MapMetadataStore(dir) + private val tagStore = new InMemoryTagStore override def metadataStore: MetadataStore = meta @@ -166,6 +168,36 @@ class FileProvider(dir:String) extends Provider with LazyLogging { metadataStore.remove(bucket) } + override def deleteObjectTagging(bucket:String, key:String): Unit = { + val bucketFile = File(s"$dir/$bucket") + val file = File(s"$dir/$bucket/$key") + logger.debug(s"reading object for s://$bucket/$key") + if (!bucketFile.exists) throw NoSuchBucketException(bucket) + if (!file.exists) throw NoSuchKeyException(bucket, key) + if (file.isDirectory) throw NoSuchKeyException(bucket, key) + tagStore.delete(bucket, key) + } + + override def getObjectTagging(bucket:String, key:String): GetObjectTagging = { + val bucketFile = File(s"$dir/$bucket") + val file = File(s"$dir/$bucket/$key") + logger.debug(s"reading object for s://$bucket/$key") + if (!bucketFile.exists) throw NoSuchBucketException(bucket) + if (!file.exists) throw NoSuchKeyException(bucket, key) + if (file.isDirectory) throw NoSuchKeyException(bucket, key) + GetObjectTagging(tagStore.get(bucket, key).getOrElse(List.empty)) + } + + override def setObjectTagging(bucket:String, key:String, tags: List[Tag]): Unit = { + val bucketFile = File(s"$dir/$bucket") + val file = File(s"$dir/$bucket/$key") + logger.debug(s"reading object for s://$bucket/$key") + if (!bucketFile.exists) throw NoSuchBucketException(bucket) + if (!file.exists) throw NoSuchKeyException(bucket, key) + if (file.isDirectory) throw NoSuchKeyException(bucket, key) + tagStore.set(bucket, key, tags) + } + /** Replace the os separator with a '/' */ private def fromOs(path: String): String = { path.replace(JFile.separatorChar, '/') diff --git a/src/main/scala/io/findify/s3mock/provider/InMemoryProvider.scala b/src/main/scala/io/findify/s3mock/provider/InMemoryProvider.scala index a355802..12f4c8e 100644 --- a/src/main/scala/io/findify/s3mock/provider/InMemoryProvider.scala +++ b/src/main/scala/io/findify/s3mock/provider/InMemoryProvider.scala @@ -4,10 +4,11 @@ import java.time.Instant import java.util.{Date, UUID} import akka.http.scaladsl.model.DateTime -import com.amazonaws.services.s3.model.ObjectMetadata +import com.amazonaws.services.s3.model.{ObjectMetadata, Tag} import com.typesafe.scalalogging.LazyLogging import io.findify.s3mock.error.{NoSuchBucketException, NoSuchKeyException} import io.findify.s3mock.provider.metadata.{InMemoryMetadataStore, MetadataStore} +import io.findify.s3mock.provider.tags.InMemoryTagStore import io.findify.s3mock.request.{CompleteMultipartUpload, CreateBucketConfiguration} import io.findify.s3mock.response._ import org.apache.commons.codec.digest.DigestUtils @@ -17,9 +18,11 @@ import scala.collection.mutable import scala.util.Random class InMemoryProvider extends Provider with LazyLogging { + private val mdStore = new InMemoryMetadataStore private val bucketDataStore = new TrieMap[String, BucketContents] private val multipartTempStore = new TrieMap[String, mutable.SortedSet[MultipartChunk]] + private val tagStore = new InMemoryTagStore private case class BucketContents(creationTime: DateTime, keysInBucket: mutable.Map[String, KeyContents]) @@ -186,4 +189,41 @@ class InMemoryProvider extends Provider with LazyLogging { case None => throw NoSuchBucketException(bucket) } } + + override def deleteObjectTagging(bucket: String, key: String): Unit = { + bucketDataStore.get(bucket) match { + case Some(bucketContent) => bucketContent.keysInBucket.get(key) match { + case Some(keyContent) => + logger.debug(s"removing tags for s://$bucket/$key") + tagStore.delete(bucket, key) + case None => throw NoSuchKeyException(bucket, key) + } + case None => throw NoSuchBucketException(bucket) + } + } + + override def getObjectTagging(bucket: String, key: String): GetObjectTagging = { + bucketDataStore.get(bucket) match { + case Some(bucketContent) => bucketContent.keysInBucket.get(key) match { + case Some(keyContent) => + logger.debug(s"reading tags for s://$bucket/$key") + GetObjectTagging(tagStore.get(bucket, key).getOrElse(List.empty)) + case None => throw NoSuchKeyException(bucket, key) + } + case None => throw NoSuchBucketException(bucket) + } + } + + override def setObjectTagging(bucket: String, key: String, tags: List[Tag]): Unit = { + bucketDataStore.get(bucket) match { + case Some(bucketContent) => bucketContent.keysInBucket.get(key) match { + case Some(keyContent) => + logger.debug(s"setting tags for s://$bucket/$key") + tagStore.set(bucket, key, tags) + case None => throw NoSuchKeyException(bucket, key) + } + case None => throw NoSuchBucketException(bucket) + } + } + } diff --git a/src/main/scala/io/findify/s3mock/provider/Provider.scala b/src/main/scala/io/findify/s3mock/provider/Provider.scala index 4dc97c7..91b7964 100644 --- a/src/main/scala/io/findify/s3mock/provider/Provider.scala +++ b/src/main/scala/io/findify/s3mock/provider/Provider.scala @@ -1,6 +1,6 @@ package io.findify.s3mock.provider -import com.amazonaws.services.s3.model.ObjectMetadata +import com.amazonaws.services.s3.model.{CompleteMultipartUploadResult ⇒ _, CopyObjectResult ⇒ _, InitiateMultipartUploadResult ⇒ _, _} import io.findify.s3mock.provider.metadata.MetadataStore import io.findify.s3mock.request.{CompleteMultipartUpload, CreateBucketConfiguration} import io.findify.s3mock.response._ @@ -25,6 +25,11 @@ trait Provider { def deleteBucket(bucket:String):Unit def copyObject(sourceBucket: String, sourceKey: String, destBucket: String, destKey: String, newMeta: Option[ObjectMetadata] = None): CopyObjectResult def copyObjectMultipart(sourceBucket: String, sourceKey: String, destBucket: String, destKey: String, partNumber:Int, uploadId:String, fromByte: Int, toByte:Int, meta: Option[ObjectMetadata] = None): CopyObjectResult + + def deleteObjectTagging(bucket:String, key:String): Unit + def getObjectTagging(bucket:String, key:String): GetObjectTagging + def setObjectTagging(bucket:String, key:String, tags: List[Tag]): Unit + } diff --git a/src/main/scala/io/findify/s3mock/provider/tags/InMemoryTagStore.scala b/src/main/scala/io/findify/s3mock/provider/tags/InMemoryTagStore.scala new file mode 100644 index 0000000..71bcb68 --- /dev/null +++ b/src/main/scala/io/findify/s3mock/provider/tags/InMemoryTagStore.scala @@ -0,0 +1,27 @@ +package io.findify.s3mock.provider.tags + +import com.amazonaws.services.s3.model.{ObjectMetadata, Tag} + +import scala.collection.concurrent.TrieMap +import scala.collection.mutable + +class InMemoryTagStore extends TagStore { + + private val bucketTags = new TrieMap[String, mutable.Map[String, List[Tag]]] + + override def delete(bucket: String, key: String): Unit = { + val currentTags = bucketTags.get(bucket) + currentTags.flatMap(_.remove(key)) + } + + override def get(bucket: String, key: String): Option[List[Tag]] = + bucketTags.get(bucket).flatMap(_.get(key)) + + override def set(bucket: String, key: String, tags: List[Tag]): Unit = { + val currentTags = bucketTags.getOrElseUpdate(bucket, new TrieMap[String, List[Tag]]()) + currentTags.put(key, tags) + } + + override def remove(bucket: String): Unit = bucketTags.remove(bucket) + +} diff --git a/src/main/scala/io/findify/s3mock/provider/tags/TagStore.scala b/src/main/scala/io/findify/s3mock/provider/tags/TagStore.scala new file mode 100644 index 0000000..06029d2 --- /dev/null +++ b/src/main/scala/io/findify/s3mock/provider/tags/TagStore.scala @@ -0,0 +1,10 @@ +package io.findify.s3mock.provider.tags + +import com.amazonaws.services.s3.model.Tag + +trait TagStore { + def delete(bucket: String, key: String): Unit + def get(bucket: String, key: String): Option[List[Tag]] + def remove(bucket: String): Unit + def set(bucket: String, key: String, tags: List[Tag]): Unit +} diff --git a/src/main/scala/io/findify/s3mock/response/GetObjectTagging.scala b/src/main/scala/io/findify/s3mock/response/GetObjectTagging.scala new file mode 100644 index 0000000..75c7c0e --- /dev/null +++ b/src/main/scala/io/findify/s3mock/response/GetObjectTagging.scala @@ -0,0 +1,20 @@ +package io.findify.s3mock.response + +import com.amazonaws.services.s3.model.Tag + +case class GetObjectTagging(tags: List[Tag]) { + def toXML = + + + { + tags.map(tag => + + {tag.getKey} + {tag.getValue} + + ) + } + + +} + diff --git a/src/main/scala/io/findify/s3mock/route/DeleteObject.scala b/src/main/scala/io/findify/s3mock/route/DeleteObject.scala index dcc9213..5a4da06 100644 --- a/src/main/scala/io/findify/s3mock/route/DeleteObject.scala +++ b/src/main/scala/io/findify/s3mock/route/DeleteObject.scala @@ -13,19 +13,29 @@ import scala.util.{Failure, Success, Try} */ case class DeleteObject(implicit provider: Provider) extends LazyLogging { def route(bucket:String, path:String) = delete { - complete { - Try(provider.deleteObject(bucket, path)) match { - case Success(_) => - logger.info(s"deleted object $bucket/$path") - HttpResponse(StatusCodes.NoContent) - case Failure(NoSuchKeyException(_, _)) => - logger.info(s"cannot delete object $bucket/$path: no such key") - HttpResponse(StatusCodes.NotFound) - case Failure(ex) => - logger.error(s"cannot delete object $bucket/$path", ex) - HttpResponse(StatusCodes.NotFound) + parameter('tagging?) { (tagging) ⇒ + tagging match { + case Some(_) ⇒ + complete { + handleTry(bucket, path, Try(provider.deleteObjectTagging(bucket, path))) + } + case None ⇒ + complete { + handleTry(bucket, path, Try(provider.deleteObject(bucket, path))) + } } - } } -} + + private def handleTry[A](bucket: String, path: String, t: Try[A]) = t match { + case Success(_) => + logger.info(s"deleted object $bucket/$path") + HttpResponse(StatusCodes.NoContent) + case Failure(NoSuchKeyException(_, _)) => + logger.info(s"cannot delete object $bucket/$path: no such key") + HttpResponse(StatusCodes.NotFound) + case Failure(ex) => + logger.error(s"cannot delete object $bucket/$path", ex) + HttpResponse(StatusCodes.NotFound) + } +} \ No newline at end of file diff --git a/src/main/scala/io/findify/s3mock/route/GetObject.scala b/src/main/scala/io/findify/s3mock/route/GetObject.scala index 1b15881..aedd017 100644 --- a/src/main/scala/io/findify/s3mock/route/GetObject.scala +++ b/src/main/scala/io/findify/s3mock/route/GetObject.scala @@ -14,6 +14,7 @@ import com.amazonaws.util.DateUtils import com.typesafe.scalalogging.LazyLogging import io.findify.s3mock.error.{InternalErrorException, NoSuchBucketException, NoSuchKeyException} import io.findify.s3mock.provider.{GetObjectData, Provider} +import io.findify.s3mock.response.GetObjectTagging import scala.collection.JavaConverters._ import scala.util.{Failure, Success, Try} @@ -38,7 +39,7 @@ case class GetObject(implicit provider: Provider) extends LazyLogging { } if (params.contains("tagging")) { - handleTaggingRequest(meta) + handleTaggingRequest(provider.getObjectTagging(bucket, path), meta) } else { HttpResponse( status = StatusCodes.OK, @@ -77,44 +78,46 @@ case class GetObject(implicit provider: Provider) extends LazyLogging { - protected def handleTaggingRequest(meta: ObjectMetadata): HttpResponse = { + protected def handleTaggingRequest(tagsMeta: GetObjectTagging, meta: ObjectMetadata): HttpResponse = { var root = var tagset = var w = new StringWriter() - if (meta.getRawMetadata.containsKey("x-amz-tagging")){ - var doc = - - - { - meta.getRawMetadata.get("x-amz-tagging").asInstanceOf[String].split("&").map( - (rawTag: String) => { - rawTag.split("=", 2).map( - (part: String) => URLDecoder.decode(part, "UTF-8") - ) - }).map( - (kv: Array[String]) => - - {kv(0)} - {kv(1)} - ) + var doc = + + + { + tagsMeta.tags.map(tag => + + {tag.getKey} + {tag.getValue} + + ) + } + { + if (meta.getRawMetadata.containsKey("x-amz-tagging")){ + meta.getRawMetadata.get("x-amz-tagging").asInstanceOf[String].split("&").map( + (rawTag: String) => { + rawTag.split("=", 2).map( + (part: String) => URLDecoder.decode(part, "UTF-8") + ) + }).map( + (kv: Array[String]) => + + {kv(0)} + {kv(1)} + ) } - - + } + + + xml.XML.write(w, doc, "UTF-8", true, null) - xml.XML.write(w, doc, "UTF-8", true, null) - } else { - var doc = - xml.XML.write(w, doc, "UTF-8", true, null) - } - - meta.setContentType("application/xml; charset=utf-8") HttpResponse( status = StatusCodes.OK, - entity = w.toString, - headers = `Last-Modified`(DateTime(1970, 1, 1)) :: metadataToHeaderList(meta) + entity = w.toString ) } diff --git a/src/main/scala/io/findify/s3mock/route/PutObject.scala b/src/main/scala/io/findify/s3mock/route/PutObject.scala index 0eb2534..1ea93e4 100644 --- a/src/main/scala/io/findify/s3mock/route/PutObject.scala +++ b/src/main/scala/io/findify/s3mock/route/PutObject.scala @@ -1,11 +1,13 @@ package io.findify.s3mock.route +import java.nio.charset.StandardCharsets + import akka.http.scaladsl.model.{HttpRequest, HttpResponse, StatusCodes} import akka.http.scaladsl.server.Directives._ import akka.stream.Materializer import akka.stream.scaladsl.Sink import akka.util.ByteString -import com.amazonaws.services.s3.model.ObjectMetadata +import com.amazonaws.services.s3.model.{ObjectMetadata, Tag} import com.typesafe.scalalogging.LazyLogging import io.findify.s3mock.S3ChunkedProtocolStage import io.findify.s3mock.error.{InternalErrorException, NoSuchBucketException} @@ -13,19 +15,31 @@ import io.findify.s3mock.provider.Provider import org.apache.commons.codec.digest.DigestUtils import scala.util.{Failure, Success, Try} +import scala.xml.XML /** * Created by shutty on 8/20/16. */ case class PutObject(implicit provider:Provider, mat:Materializer) extends LazyLogging { def route(bucket:String, path:String) = put { - extractRequest { request => - headerValueByName("authorization") { auth => - completeSigned(bucket, path) - } ~ completePlain(bucket, path) + parameter('tagging?) { (tagging) ⇒ + tagging match { + case Some(_) ⇒ completePutTags(bucket, path) + case None ⇒ + extractRequest { request => + headerValueByName("authorization") { auth => + completeSigned(bucket, path) + } ~ completePlain(bucket, path) + } + } } } ~ post { - completePlain(bucket, path) + parameter('tagging?) { (tagging) ⇒ + tagging match { + case Some(_) ⇒ completePutTags(bucket, path) + case None ⇒ completePlain(bucket, path) + } + } } @@ -85,6 +99,38 @@ case class PutObject(implicit provider:Provider, mat:Materializer) extends LazyL } } + def completePutTags(bucket:String, path:String) = extractRequest { request ⇒ + complete { + logger.info(s"put object $bucket/$path (unsigned)") + val result = request.entity.dataBytes + .fold(ByteString(""))(_ ++ _) + .map(data => { + + val tags = (for { + tag ← XML.loadString(new String(data.toArray, StandardCharsets.UTF_8)) \ "TagSet" \ "Tag" + key ← tag \ "Key" + value ← tag \ "Value" + } yield new Tag(key.text, value.text)).toList + + Try(provider.setObjectTagging(bucket, path, tags)) match { + case Success(()) => HttpResponse(StatusCodes.OK) + case Failure(e: NoSuchBucketException) => + HttpResponse( + StatusCodes.NotFound, + entity = e.toXML.toString() + ) + case Failure(t) => + HttpResponse( + StatusCodes.InternalServerError, + entity = InternalErrorException(t).toXML.toString() + ) + } + + }).runWith(Sink.head[HttpResponse]) + result + } + } + private def populateObjectMetadata(request: HttpRequest, bytes: Array[Byte]): ObjectMetadata = { val metadata = MetadataUtil.populateObjectMetadata(request) metadata.setContentMD5(DigestUtils.md5Hex(bytes)) diff --git a/src/test/scala/io/findify/s3mock/GetPutObjectTest.scala b/src/test/scala/io/findify/s3mock/GetPutObjectTest.scala index d751a8e..c70a20a 100644 --- a/src/test/scala/io/findify/s3mock/GetPutObjectTest.scala +++ b/src/test/scala/io/findify/s3mock/GetPutObjectTest.scala @@ -70,6 +70,7 @@ class GetPutObjectTest extends S3MockTest { tagMap.get("key1") shouldBe "val1" tagMap.get("key=&interesting") shouldBe "value=something&stragne" } + it should "be OK with retrieving tags for un-tagged objects" in { s3.putObject("tbucket", "taggedobj", "some-content") var tagging = s3.getObjectTagging(new GetObjectTaggingRequest("tbucket", "taggedobj")).getTagSet diff --git a/src/test/scala/io/findify/s3mock/ObjectTaggingTest.scala b/src/test/scala/io/findify/s3mock/ObjectTaggingTest.scala new file mode 100644 index 0000000..828a2af --- /dev/null +++ b/src/test/scala/io/findify/s3mock/ObjectTaggingTest.scala @@ -0,0 +1,45 @@ +package io.findify.s3mock + +import java.io.ByteArrayInputStream +import java.util + +import com.amazonaws.services.s3.model._ + +class ObjectTaggingTest extends S3MockTest { + override def behaviour(fixture: => Fixture): Unit = { + val s3 = fixture.client + it should "handle PUT / GET / DELETE tagging requests" in { + + s3.createBucket("tbucket") + s3.putObject( + new PutObjectRequest("tbucket", "taggedobj", new ByteArrayInputStream("content".getBytes("UTF-8")), new ObjectMetadata) + ) + + import scala.collection.JavaConverters._ + s3.setObjectTagging( + new SetObjectTaggingRequest( + "tbucket", + "taggedobj", + new ObjectTagging(List(new Tag("key1", "val1"), new Tag("key=&interesting", "value=something&stragne")).asJava))) + + var tagging1 = s3.getObjectTagging(new GetObjectTaggingRequest("tbucket", "taggedobj")).getTagSet.asScala + var tagMap1 = new util.HashMap[String, String]() + for (tag <- tagging1) { + tagMap1.put(tag.getKey, tag.getValue) + } + tagMap1.size() shouldBe 2 + tagMap1.get("key1") shouldBe "val1" + tagMap1.get("key=&interesting") shouldBe "value=something&stragne" + + s3.deleteObjectTagging(new DeleteObjectTaggingRequest("tbucket", "taggedobj")) + + var tagging2 = s3.getObjectTagging(new GetObjectTaggingRequest("tbucket", "taggedobj")).getTagSet.asScala + var tagMap2 = new util.HashMap[String, String]() + for (tag <- tagging2) { + tagMap2.put(tag.getKey, tag.getValue) + } + tagMap2.size() shouldBe 0 + + } + } +}