package com.example.aiapp import android.content.Context import android.graphics.Bitmap import android.graphics.BitmapFactory import android.graphics.Matrix import android.net.Uri import androidx.exifinterface.media.ExifInterface import java.io.ByteArrayOutputStream import kotlin.math.max /** * Getting a picked photo to a session, at a size the session can actually take. * * A phone camera produces twelve megapixels and several megabytes. The Claude API resizes anything * larger than 1568px on its long edge before looking at it and refuses images past a much higher * bound outright, so a photo sent straight off the camera roll was uploaded whole over the tunnel * to be either thrown away or rejected -- which is what "sending an image is broken" was. * * Shrunk here rather than on the backend, so the bytes that never mattered are never sent: the * expensive part of this on a phone is the upload, not the decode. What the limit *is* comes from * the server, per session -- see `DriverKind::max_image_edge` -- because that is where a provider's * requirements are known, and a phone that carried its own copy of them would be a second place to * update when one changes. */ suspend fun uploadPickedImage( context: Context, settings: ServerSettings, sessionId: String, uri: Uri, maxEdge: Int?, ): String { val (bytes, mime) = readForUpload(context, uri, maxEdge) return uploadAttachment(settings, sessionId, bytes, mime) } /** * The bytes to upload and what they are, scaled down only if they need to be. * * An image already inside the limit is uploaded exactly as it came, rather than decoded and * re-encoded to the same size: a round trip through JPEG loses a little every time, and there is * nothing to gain from it. This is also the path a provider with no limit always takes. */ private fun readForUpload(context: Context, uri: Uri, maxEdge: Int?): Pair { val resolver = context.contentResolver val mime = resolver.getType(uri) ?: "image/jpeg" val original = resolver.openInputStream(uri)?.use { it.readBytes() } ?: throw ApiException("couldn't read the picked image") if (maxEdge == null) return original to mime val bounds = BitmapFactory.Options().apply { inJustDecodeBounds = true } BitmapFactory.decodeByteArray(original, 0, original.size, bounds) val longest = max(bounds.outWidth, bounds.outHeight) // outWidth is -1 when the bytes are not an image this device can decode. Sent on untouched: // this function's job is the size, and refusing something the server might understand is a // decision it has no business making. if (longest <= 0 || longest <= maxEdge) return original to mime // Powers of two first, which is all the decoder can do, and then the exact scale. Decoding // the full twelve megapixels only to shrink it is how this runs out of memory on the images // it most needs to handle. val decode = BitmapFactory.Options().apply { inSampleSize = Integer.highestOneBit(max(1, longest / maxEdge)) } val decoded = BitmapFactory.decodeByteArray(original, 0, original.size, decode) ?: return original to mime val scale = maxEdge.toFloat() / max(decoded.width, decoded.height) val matrix = Matrix() if (scale < 1f) matrix.postScale(scale, scale) // The camera writes which way up the picture is into EXIF rather than rotating the pixels, and // re-encoding drops the tag -- so a portrait photo would arrive at the model on its side, with // nothing anywhere saying so. Applied to the same matrix as the scale, so it costs no second // copy of the bitmap. matrix.postRotate(exifRotation(original)) val scaled = Bitmap.createBitmap(decoded, 0, 0, decoded.width, decoded.height, matrix, true) val out = ByteArrayOutputStream() // JPEG whatever came in: this is a photograph being made smaller, which is what JPEG is for, // and a PNG of a resampled photo is several times the size for no visible difference. scaled.compress(Bitmap.CompressFormat.JPEG, JPEG_QUALITY, out) return out.toByteArray() to "image/jpeg" } /** How far to turn the picture so it is the way up it was taken. */ private fun exifRotation(bytes: ByteArray): Float = try { when ( ExifInterface(bytes.inputStream()) .getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL) ) { ExifInterface.ORIENTATION_ROTATE_90 -> 90f ExifInterface.ORIENTATION_ROTATE_180 -> 180f ExifInterface.ORIENTATION_ROTATE_270 -> 270f else -> 0f } } catch (_: java.io.IOException) { // No EXIF, or none this can read. Upright is the assumption every // image without the tag is displayed under anyway. 0f } /** High enough that resampling is what the reader notices, not the encoder. */ private const val JPEG_QUALITY = 90