package com.example.aiapp import android.content.ContentResolver import android.content.Context import android.graphics.Bitmap import android.graphics.BitmapFactory import android.graphics.Matrix import android.net.Uri import android.provider.OpenableColumns import androidx.exifinterface.media.ExifInterface import java.io.ByteArrayOutputStream import kotlin.math.max /** * Getting a picked photo to a session, at a size the session can actually take. * * A phone camera produces twelve megapixels and several megabytes. The Claude API resizes anything * larger than 1568px on its long edge before looking at it and refuses images past a much higher * bound outright, so a photo sent straight off the camera roll was uploaded whole over the tunnel * to be either thrown away or rejected -- which is what "sending an image is broken" was. * * Shrunk here rather than on the backend, so the bytes that never mattered are never sent: the * expensive part on a phone is the upload, not the decode. What the limit *is* comes from the * server, per session, because that is where a provider's requirements are known. */ suspend fun uploadPickedImage( context: Context, settings: ServerSettings, sessionId: String, uri: Uri, maxEdge: Int?, ): String { val (bytes, mime) = readForUpload(context, uri, maxEdge) return uploadAttachment(settings, sessionId, mime, "image") { it.write(bytes) } } /** * Uploads whatever [uri] names, the way its kind needs. An image goes through [uploadPickedImage] * and is shrunk; anything else goes whole, under the name the other app or the file chooser gave * it, because the session is told that name rather than shown the bytes. */ suspend fun uploadPicked( context: Context, settings: ServerSettings, sessionId: String, uri: Uri, maxEdge: Int?, ): String { val resolver = context.contentResolver val mime = resolver.getType(uri) if (mime != null && mime.startsWith("image/")) { return uploadPickedImage(context, settings, sessionId, uri, maxEdge) } // Opened before the request starts, so a provider that refuses says so here and not from inside // the connection; then streamed, since a trace is bigger than this process should hold at once. val source = openSource(resolver, uri) val name = displayName(resolver, uri) return uploadAttachment(settings, sessionId, mime ?: "application/octet-stream", name) { out -> try { source.use { it.copyTo(out, COPY_BUFFER) } } catch (e: java.io.IOException) { // Either side of the copy can fail; the message names the file, which is the part the // reader can do something about. throw ApiException("couldn't send $name: ${e.message}", cause = e) } } } private const val COPY_BUFFER = 64 * 1024 /** * A stream of [uri], or the refusal as the kind the composer reports beside the message. * * A share arrives with whatever access the other app granted, and a provider that refuses says so * with a `SecurityException`; a file gone between the pick and the read is an `IOException`. Both * are things the reader can act on. */ private fun openSource(resolver: ContentResolver, uri: Uri): java.io.InputStream = try { resolver.openInputStream(uri) ?: throw ApiException("couldn't read ${uri.lastPathSegment ?: uri}: nothing there") } catch (e: SecurityException) { throw ApiException("couldn't read ${uri.lastPathSegment ?: uri}: no access to it") } catch (e: java.io.IOException) { throw ApiException("couldn't read ${uri.lastPathSegment ?: uri}: ${e.message}") } /** Everything at [uri]; an image is decoded whole anyway, so it is read whole. */ private fun readAll(resolver: ContentResolver, uri: Uri): ByteArray = openSource(resolver, uri).use { it.readBytes() } /** * The name a document provider shows for [uri]. The last path segment is the fallback because a * provider's own id for a file is usually a number, which says nothing to the session. */ private fun displayName(resolver: ContentResolver, uri: Uri): String { resolver.query(uri, arrayOf(OpenableColumns.DISPLAY_NAME), null, null, null)?.use { cursor -> val column = cursor.getColumnIndex(OpenableColumns.DISPLAY_NAME) if (column >= 0 && cursor.moveToFirst()) cursor.getString(column)?.let { return it } } return uri.lastPathSegment ?: "file" } /** * The bytes to upload and what they are, scaled down only if they need to be. * * An image already inside the limit is uploaded exactly as it came, rather than decoded and re- * encoded to the same size: a round trip through JPEG loses a little every time. This is also the * path a provider with no limit always takes. */ private fun readForUpload(context: Context, uri: Uri, maxEdge: Int?): Pair { val resolver = context.contentResolver val mime = resolver.getType(uri) ?: "image/jpeg" val original = readAll(resolver, uri) if (maxEdge == null) return original to mime val bounds = BitmapFactory.Options().apply { inJustDecodeBounds = true } BitmapFactory.decodeByteArray(original, 0, original.size, bounds) val longest = max(bounds.outWidth, bounds.outHeight) // outWidth is -1 when the bytes are not an image this device can decode. Sent on untouched: // this function's job is the size, and refusing something the server might understand is a // decision it has no business making. if (longest <= 0 || longest <= maxEdge) return original to mime // Powers of two first, which is all the decoder can do, and then the exact scale. Decoding the // full twelve megapixels only to shrink it is how this runs out of memory on the images it most // needs to handle. val decode = BitmapFactory.Options().apply { inSampleSize = Integer.highestOneBit(max(1, longest / maxEdge)) } val decoded = BitmapFactory.decodeByteArray(original, 0, original.size, decode) ?: return original to mime val scale = maxEdge.toFloat() / max(decoded.width, decoded.height) val matrix = Matrix() if (scale < 1f) matrix.postScale(scale, scale) // The camera writes which way up the picture is into EXIF rather than rotating the pixels, and // re-encoding drops the tag -- so a portrait photo would arrive at the model on its side. // Applied to the same matrix as the scale, so it costs no second copy of the bitmap. matrix.postRotate(exifRotation(original)) val scaled = Bitmap.createBitmap(decoded, 0, 0, decoded.width, decoded.height, matrix, true) val out = ByteArrayOutputStream() // JPEG whatever came in: this is a photograph being made smaller, which is what JPEG is for, // and a PNG of a resampled photo is several times the size for no visible difference. scaled.compress(Bitmap.CompressFormat.JPEG, JPEG_QUALITY, out) return out.toByteArray() to "image/jpeg" } /** How far to turn the picture so it is the way up it was taken. */ private fun exifRotation(bytes: ByteArray): Float = try { when ( ExifInterface(bytes.inputStream()) .getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL) ) { ExifInterface.ORIENTATION_ROTATE_90 -> 90f ExifInterface.ORIENTATION_ROTATE_180 -> 180f ExifInterface.ORIENTATION_ROTATE_270 -> 270f else -> 0f } } catch (_: java.io.IOException) { // No EXIF, or none this can read. Upright is the assumption every image without the tag is // displayed under anyway. 0f } /** High enough that resampling is what the reader notices, not the encoder. */ private const val JPEG_QUALITY = 90