Added new JS5 file server (launcher no longer has to handle cache downloading) - PLEASE SET YOUR CACHE LOCATION IN file-server.properties

Updated client to connect to the new file server

.gitignore

@@ -14,6 +14,11 @@
 /Server/build/kotlin/
 /Server/build/generated/
 /Server/data/eco/bot_offers.json
+/File-Server/build/
+/File-Server/build/classes/
+/File-Server/build/tmp/
+/File-Server/build/kotlin/
+/File-Server/build/generated/
 **/.idea/workspace.xml
 **/.idea/tasks.xml
 Server/**/*.class

File-Server/build.gradle

@@ -0,0 +1,37 @@
+apply plugin: 'application'
+
+archivesBaseName = 'fileserver'
+
+mainClassName = 'js5server.JS5Server'
+
+group 'org.rs09'
+version '1.0.0'
+
+repositories {
+    mavenCentral()
+}
+
+dependencies {
+
+    implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.4.2'
+    /* Networking */
+    implementation "io.ktor:ktor-server-core:1.5.0"
+    implementation "io.ktor:ktor-network:1.5.0"
+
+    // Cache ops
+    implementation 'com.displee:rs-cache-library:6.8'
+
+    testImplementation 'org.junit.jupiter:junit-jupiter-api:5.6.0'
+    testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine'
+}
+
+test {
+    useJUnitPlatform()
+}
+
+jar {
+    manifest {
+        attributes 'Main-Class': 'js5server.JS5Server'
+    }
+    from { configurations.compileClasspath.collect { it.isDirectory() ? it : zipTree(it) } }
+}

File-Server/file-server.properties

@@ -0,0 +1,4 @@
+revision=530
+subrevision=1
+port=43593
+cachePath=

File-Server/src/main/kotlin/js5server/DataProvider.kt

@@ -0,0 +1,18 @@
+package js5server
+
+import com.displee.cache.CacheLibrary
+
+interface DataProvider {
+    fun data(index: Int, archive: Int): ByteArray?
+
+    companion object {
+        operator fun invoke(cache: CacheLibrary) = object : DataProvider {
+            override fun data(index: Int, archive: Int) =
+                if (index == 255) {
+                    cache.index255?.readArchiveSector(archive)?.data
+                } else {
+                    cache.index(index).readArchiveSector(archive)?.data
+                }
+        }
+    }
+}

File-Server/src/main/kotlin/js5server/FileServer.kt

@@ -0,0 +1,111 @@
+package js5server
+
+import com.displee.cache.CacheLibrary
+import io.ktor.utils.io.*
+import js5server.ext.readUMedium
+import kotlin.math.min
+
+class FileServer(
+    private val provider: DataProvider,
+    private val versionTable: ByteArray
+) {
+
+    /**
+     * Fulfills a request by sending the requested files data to the requester
+     */
+    suspend fun fulfill(read: ByteReadChannel, write: ByteWriteChannel, prefetch: Boolean) {
+        val value = read.readUMedium()
+        val index = value shr 16
+        val archive = value and 0xffff
+        val data = data(index, archive) ?: return println("Unable to fulfill request $index $archive $prefetch.")
+        if (index == 255 && archive == 255) {
+            serve255(write, data)
+        } else {
+            serve(write, index, archive, data, prefetch)
+        }
+    }
+
+    /**
+     * @return data for an [index]'s [archive] file or [versionTable] when index and archive are both 255
+     */
+    fun data(index: Int, archive: Int): ByteArray? {
+        if (index == 255 && archive == 255) {
+            return versionTable
+        }
+        return provider.data(index, archive)
+    }
+
+    /**
+     * Writes [source] [offset] [size] to [write] and starting at [headerSize] inserting a [SEPARATOR] every [split] bytes
+     */
+    private suspend fun serve255(write: ByteWriteChannel, source: ByteArray) {
+        write.writeByte(255)
+        write.writeShort(255)
+        write.writeByte(0)
+        write.writeInt(source.size)
+        write.writeFully(source)
+    }
+
+    /**
+     * Writes response header followed by the contents of [data] to [write]
+     */
+    suspend fun serve(write: ByteWriteChannel, index: Int, archive: Int, data: ByteArray, prefetch: Boolean) {
+        val compression = data[0].toInt()
+        val size = getInt(data[1], data[2], data[3], data[4]) + if (compression != 0) 8 else 4
+        write.writeByte(index)
+        write.writeShort(archive)
+        write.writeByte(if (prefetch) compression or 0x80 else compression)
+        serve(write, HEADER, data, OFFSET, size, SPLIT)
+    }
+
+    /**
+     * Writes [source] [offset] [size] to [write] and starting at [headerSize] inserting a [SEPARATOR] every [split] bytes
+     */
+    suspend fun serve(write: ByteWriteChannel, headerSize: Int, source: ByteArray, offset: Int, size: Int, split: Int) {
+        var length = min(size, split - headerSize)
+        write.writeFully(source, offset, length)
+        var written = length
+        while (written < size) {
+            write.writeByte(SEPARATOR)
+
+            length = if (size - written < split) size - written else split - 1
+            write.writeFully(source, written + offset, length)
+            written += length
+        }
+    }
+
+    companion object {
+
+        private fun getInt(b1: Byte, b2: Byte, b3: Byte, b4: Byte) = b1.toInt() shl 24 or (b2.toInt() and 0xff shl 16) or (b3.toInt() and 0xff shl 8) or (b4.toInt() and 0xff)
+
+        private const val SEPARATOR = 255
+        private const val HEADER = 4
+        private const val SPLIT = 512
+        private const val OFFSET = 1
+    }
+
+    /**
+     *  Length of archive with [name] in [index]
+     */
+    fun file(cacheLibrary: CacheLibrary, index: Int, name: String): Int {
+        val idx = cacheLibrary.index(index)
+        val archive = idx.archiveId(name)
+        if (archive == -1) {
+            return 0
+        }
+        return (idx.readArchiveSector(archive)?.size ?: 2) - 2
+    }
+
+    /**
+     * Length of all archives in [index]
+     */
+    fun archive(cache: CacheLibrary, index: Int): Int {
+        var total = 0
+        val idx = cache.index(index)
+        idx.archiveIds().forEach { archive ->
+            total += idx.readArchiveSector(archive)?.size ?: 0
+        }
+        total += cache.index255?.readArchiveSector(index)?.size ?: 0
+        return total
+    }
+}

File-Server/src/main/kotlin/js5server/JS5Net.kt

@@ -0,0 +1,152 @@
+package js5server
+
+import io.ktor.network.selector.*
+import io.ktor.network.sockets.*
+import io.ktor.utils.io.*
+import js5server.ext.readMedium
+import kotlinx.coroutines.*
+import java.util.concurrent.Executors
+
+class JS5Net(
+    private val server: FileServer,
+    private val revision: Int,
+    private val subRevision: Int
+) {
+
+    private val exceptionHandler = CoroutineExceptionHandler { context, throwable ->
+        System.err.println("$context | $throwable")
+    }
+
+    private lateinit var dispatcher: ExecutorCoroutineDispatcher
+    private var running = false
+
+    /**
+     * Start the server and begin creating a new coroutine for every new connection accepted
+     * @param threads a fixed number or 0 to dynamically allocate based on need
+     */
+    fun start(port: Int, threads: Int) = runBlocking {
+        val executor = if (threads == 0) Executors.newCachedThreadPool() else Executors.newFixedThreadPool(threads)
+        dispatcher = executor.asCoroutineDispatcher()
+        val selector = ActorSelectorManager(dispatcher)
+        val supervisor = SupervisorJob()
+        val scope = CoroutineScope(coroutineContext + supervisor + exceptionHandler)
+        with(scope) {
+            val server = aSocket(selector).tcp().bind(port = port)
+            running = true
+            while (running) {
+                val socket = server.accept()
+                launch(Dispatchers.IO) {
+                    connect(socket)
+                }
+            }
+        }
+    }
+
+    suspend fun connect(socket: Socket) {
+        val read = socket.openReadChannel()
+        val write = socket.openWriteChannel(autoFlush = true)
+        synchronise(read, write)
+        if (acknowledge(read, write)) {
+            readRequests(read, write)
+        }
+    }
+
+    /**
+     * If the client is up-to-date and in the correct state send it the [prefetchKeys] list so it knows what indices are available to request
+     */
+    private suspend fun synchronise(read: ByteReadChannel, write: ByteWriteChannel) {
+        val opcode = read.readByte().toInt()
+        println("Received $opcode")
+        if (opcode != HANDSHAKE_REQUEST) {
+            write.writeByte(REJECT_SESSION)
+            write.close()
+            return
+        }
+
+        val revision = read.readInt()
+        val version = read.readInt()
+        println("Received ${this.revision}.${this.subRevision} | ${revision}.$version")
+        if (revision != this.revision || version != this.subRevision) {
+            write.writeByte(GAME_UPDATED)
+            write.close()
+            return
+        }
+        write.writeByte(0)
+    }
+
+    /**
+     * Confirm the client got our message and is ready to start sending file requests
+     */
+    private suspend fun acknowledge(read: ByteReadChannel, write: ByteWriteChannel): Boolean {
+        val opcode = read.readByte().toInt()
+        println("Received $opcode")
+        if (opcode != ACKNOWLEDGE) {
+            write.writeByte(REJECT_SESSION)
+            write.close()
+            return false
+        }
+
+        return verify(read, write, ACKNOWLEDGE_ID)
+    }
+
+    /**
+     * Confirm a session value send by the client is as the server [expected]
+     */
+    private suspend fun verify(read: ByteReadChannel, write: ByteWriteChannel, expected: Int): Boolean {
+        val id = read.readMedium()
+        if (id != expected) {
+            write.writeByte(BAD_SESSION_ID)
+            write.close()
+            return false
+        }
+        return true
+    }
+
+    private suspend fun readRequests(read: ByteReadChannel, write: ByteWriteChannel) = coroutineScope {
+        try {
+            while (isActive) {
+                readRequest(read, write)
+            }
+        } finally {
+            println("Client disconnected js5")
+        }
+    }
+
+    /**
+     * Verify status updates and pass requests onto the [server] to fulfill
+     */
+    private suspend fun readRequest(read: ByteReadChannel, write: ByteWriteChannel) {
+        when (val opcode = read.readByte().toInt()) {
+            STATUS_LOGGED_OUT, STATUS_LOGGED_IN -> verify(read, write, STATUS_ID)
+            PRIORITY_REQUEST, PREFETCH_REQUEST -> server.fulfill(read, write, opcode == PREFETCH_REQUEST)
+            else -> {
+                println("Closing write")
+                write.close()
+            }
+        }
+    }
+
+    fun stop() {
+        running = false
+        dispatcher.close()
+    }
+
+    companion object {
+        // Session ids
+        const val ACKNOWLEDGE_ID = 3
+        const val STATUS_ID = 0
+
+        // Opcodes
+        const val PREFETCH_REQUEST = 0
+        const val PRIORITY_REQUEST = 1
+        const val HANDSHAKE_REQUEST = 15
+        const val STATUS_LOGGED_IN = 2
+        const val STATUS_LOGGED_OUT = 3
+        const val ACKNOWLEDGE = 6
+
+        // Response codes
+        private const val GAME_UPDATED = 6
+        private const val BAD_SESSION_ID = 10
+        private const val REJECT_SESSION = 11
+    }
+}

File-Server/src/main/kotlin/js5server/JS5Server.kt

@@ -0,0 +1,52 @@
+package js5server
+
+import com.displee.cache.CacheLibrary
+import java.io.File
+import kotlin.concurrent.thread
+
+object JS5Server {
+
+    @JvmStatic
+    fun main(args: Array<String>) {
+        val start = System.currentTimeMillis()
+        val file = File("./file-server.properties")
+        if (!file.exists()) {
+            System.err.println("Unable to find server properties file.")
+            return
+        }
+
+        println("Start up...")
+
+        var revision = 530
+        var subRevision = 1
+        var port = 43593
+        var threads = 0
+        lateinit var cachePath: String
+        file.forEachLine { line ->
+            val (key, value) = line.split("=")
+            when (key) {
+                "revision" -> revision = value.toInt()
+                "subrevision" -> subRevision = value.toInt()
+                "port" -> port = value.toInt()
+                "threads" -> threads = value.toInt()
+                "cachePath" -> cachePath = value
+            }
+        }
+
+        println("Loaded configuration.")
+
+        val cache = CacheLibrary(cachePath)
+        val versionTable = cache.generateOldUkeys()
+        val fileServer = FileServer(DataProvider(cache), versionTable)
+
+        println("Loaded cache revision $revision from $cachePath")
+
+        val network = JS5Net(fileServer, revision, subRevision)
+
+        println("Loading complete [${System.currentTimeMillis() - start}ms] / bound to 127.0.0.1:$port")
+
+        val runtime = Runtime.getRuntime()
+        runtime.addShutdownHook(thread(start = false) { network.stop() })
+        network.start(port, threads)
+    }
+}

File-Server/src/main/kotlin/js5server/ext/JagexTypes.kt

@@ -0,0 +1,88 @@
+package js5server.ext
+
+import io.ktor.utils.io.*
+import io.ktor.utils.io.core.*
+import java.nio.ByteBuffer
+import kotlin.text.toByteArray
+
+private val CHARSET = charset("UTF-8")
+
+suspend fun ByteReadChannel.readMedium(): Int {
+    return (readByte().toInt() shl 16) + (readByte().toInt() shl 8) + readByte().toInt()
+}
+
+suspend fun ByteReadChannel.readUByte(): Int {
+    return readByte().toInt() and 0xff
+}
+
+suspend fun ByteReadChannel.readUMedium(): Int {
+    return (readUByte() shl 16) + (readUByte() shl 8) + readUByte()
+}
+
+fun ByteBuffer.putSmart(value: Int) {
+    if (value >= 128) {
+        putShort((value + 32768).toShort())
+    } else {
+        put(value.toByte())
+    }
+}
+
+fun ByteBuffer.putJagexString(string: String) {
+    put(0)
+    put(string.toByteArray(CHARSET))
+    put(0)
+}
+
+fun ByteBuffer.putShortA(value: Int) {
+    put(((value shr 8) and 0xFFFF).toByte())
+    put(((value + 128) and 0xFF).toByte())
+}
+
+fun ByteBuffer.putIntB(value: Int) {
+    put((value shr 16).toByte())
+    put((value shr 24).toByte())
+    put((value).toByte())
+    put((value shr 8).toByte())
+}
+
+fun ByteBuffer.putUnsignedByteS(value: Byte) {
+    put(((value + 128) and 0xFF).toByte())
+}
+
+fun ByteBuffer.putUnsignedShort(value: Int) {
+    putShort(((value) and 0xFFFF).toShort())
+}
+
+fun BytePacketBuilder.writeUnsignedByteSubtract(value: Int) {
+    writeByte(((value + 128) and 0xFF).toByte())
+}
+
+fun BytePacketBuilder.writeUnsignedShort(value: Int) {
+    writeShort(((value) and 0xFFFF).toShort())
+}
+
+fun BytePacketBuilder.writeShortAdd(value: Int) {
+    writeByte((value shr 8).toByte())
+    writeByte((value + 128).toByte())
+}
+
+fun BytePacketBuilder.writeSmart(value: Int) {
+    if (value >= 128) {
+        writeShort((value + 32768).toShort())
+    } else {
+        writeByte(value.toByte())
+    }
+}
+
+fun BytePacketBuilder.writeIntME(value: Int) {
+    writeByte((value shr 16).toByte())
+    writeByte((value shr 24).toByte())
+    writeByte(value.toByte())
+    writeByte((value shr 8).toByte())
+}
+
+fun BytePacketBuilder.writeVersionedString(value: String) {
+    writeByte(0)
+    writeText(value)
+    writeByte(0)
+}

settings.gradle

@@ -1,4 +1,5 @@
 rootProject.name = 'RS09'
 include 'Client'
 include 'Management-Server'
-include 'Server'
+include 'Server'
+include 'File-Server'