Halt tick processing if the server isn't able to reach the internet when watchdog is enabled (configurable via server config options connectivity_check_url and connectivity_timeout)

Server/src/main/core/Server.kt

@@ -3,20 +3,21 @@ package core
 import core.api.log
 import core.game.system.SystemManager
 import core.game.system.SystemState
-import core.net.NioReactor
-import core.tools.TimeStamp
-import kotlinx.coroutines.*
-import core.tools.SystemLogger
 import core.game.system.config.ServerConfigParser
 import core.game.world.GameWorld
-import core.game.world.repository.Repository
+import core.net.NioReactor
 import core.tools.Log
+import core.tools.NetworkReachability
+import core.tools.TimeStamp
+import kotlinx.coroutines.*
 import java.io.File
 import java.io.FileWriter
 import java.lang.management.ManagementFactory
 import java.lang.management.ThreadMXBean
 import java.net.BindException
+import java.net.URL
 import java.util.*
+import kotlin.math.max
 import kotlin.system.exitProcess
 
 
@@ -43,6 +44,8 @@ object Server {
     @JvmStatic
     var reactor: NioReactor? = null
 
+    var networkReachability = NetworkReachability.Reachable
+
     /**
      * The main method, in this method we load background utilities such as
      * cache and our world, then end with starting networking.
@@ -95,6 +98,11 @@ object Server {
             GlobalScope.launch {
                 delay(20000)
                 while (running) {
+                    val timeStart = System.currentTimeMillis()
+                    if (!checkConnectivity())
+                        networkReachability = NetworkReachability.Unreachable
+                    else
+                        networkReachability = NetworkReachability.Reachable
                     if (System.currentTimeMillis() - lastHeartbeat > 7200 && running) {
                         log(this::class.java, Log.ERR, "Triggering reboot due to heartbeat timeout")
                         log(this::class.java, Log.ERR, "Creating thread dump...")
@@ -115,12 +123,36 @@ object Server {
                         if (!SystemManager.isTerminated())
                             exitProcess(0)
                     }
-                    delay(625)
+                    val timeNow = System.currentTimeMillis()
+                    delay(max(0L, 625 - (timeNow - timeStart)))
                 }
             }
         }
     }
 
+    private fun checkConnectivity(): Boolean
+    {
+        //Has to be done this way because you can't actually ping in Java unless you run the whole thing as root
+        val urls = ServerConstants.CONNECTIVITY_CHECK_URL.split(",")
+        var timeout = ServerConstants.CONNECTIVITY_TIMEOUT
+        if (timeout * urls.size > 5000) //Limit timeout down to 5000ms so other watchdog functions continue as expected.
+            timeout = 5000 / urls.size
+        for (targetUrl in urls) {
+            try {
+                val url = URL(targetUrl)
+                val conn = url.openConnection()
+                conn.connectTimeout = timeout
+                conn.connect()
+                conn.getInputStream().close()
+                return true
+            } catch (e: Exception) {
+                log(this::class.java, Log.WARN, "${targetUrl} failed to respond. Are we offline?")
+                continue
+            }
+        }
+        return false
+    }
+
     @JvmStatic
     fun heartbeat() {
         lastHeartbeat = System.currentTimeMillis()

Server/src/main/core/ServerConstants.kt

@@ -338,5 +338,11 @@ class ServerConstants {
 
         @JvmField
         var STARTUP_MOMENT = Calendar.getInstance()
+
+        @JvmField
+        var CONNECTIVITY_CHECK_URL = "https://google.com,https://2009scape.org"
+
+        @JvmField
+        var CONNECTIVITY_TIMEOUT = 500
     }
 }

Server/src/main/core/game/system/config/ServerConfigParser.kt

@@ -166,6 +166,8 @@ object ServerConfigParser {
         ServerConstants.FORCE_EASTER_EVENTS = data.getBoolean("world.force_easter_randoms", false)
         ServerConstants.RUNECRAFTING_FORMULA_REVISION = data.getLong("world.runecrafting_formula_revision", 581).toInt()
         ServerConstants.ENHANCED_DEEP_WILDERNESS = data.getBoolean("world.enhanced_deep_wilderness", false)
+        ServerConstants.CONNECTIVITY_CHECK_URL = data.getString("server.connectivity_check_url", "https://google.com,https://2009scape.org")
+        ServerConstants.CONNECTIVITY_TIMEOUT = data.getLong("server.connectivity_timeout", 500L).toInt()
 
         val logLevel = data.getString("server.log_level", "VERBOSE").uppercase()
         ServerConstants.LOG_LEVEL = parseEnumEntry<LogLevel>(logLevel) ?: LogLevel.VERBOSE

Server/src/main/core/tools/NetworkReachability.kt

@@ -0,0 +1,6 @@
+package core.tools
+
+enum class NetworkReachability {
+    Reachable,
+    Unreachable
+}

Server/src/main/core/worker/MajorUpdateWorker.kt

@@ -1,25 +1,25 @@
 package core.worker
 
-import core.api.submitWorldPulse
-import core.game.system.task.Pulse
-import core.plugin.CorePluginTypes.Managers
 import core.Server
 import core.ServerConstants
 import core.ServerStore
 import core.api.log
-import core.tools.SystemLogger
+import core.api.submitWorldPulse
+import core.game.system.task.Pulse
 import core.game.world.GameWorld
 import core.game.world.repository.Repository
 import core.game.world.update.UpdateSequence
+import core.integrations.grafana.Grafana
 import core.net.packet.PacketProcessor
 import core.net.packet.PacketWriteQueue
+import core.plugin.CorePluginTypes.Managers
 import core.tools.Log
+import core.tools.NetworkReachability
 import core.tools.colorize
 import java.lang.Long.max
 import java.text.SimpleDateFormat
 import java.util.*
 import kotlin.system.exitProcess
-import core.integrations.grafana.*
 
 /**
  * Handles the running of pulses and writing of masks, etc
@@ -38,7 +38,11 @@ class MajorUpdateWorker {
             Grafana.startTick()
             val start = System.currentTimeMillis()
             Server.heartbeat()
+
+            if (Server.networkReachability == NetworkReachability.Reachable)
                 handleTickActions()
+            else
+                tickOffline()
 
             for (player in Repository.players.filter { !it.isArtificial }) {
                 if (System.currentTimeMillis() - player.session.lastPing > 20000L) {
@@ -89,6 +93,12 @@ class MajorUpdateWorker {
         log(this::class.java, Log.FINE,  "Update worker stopped.")
     }
 
+    fun tickOffline()
+    {
+        Repository.disconnectionQueue.update() //continue processing disconnection queue
+        GameWorld.pulse() //continue incrementing the global tick count
+    }
+
     fun handleTickActions(skipPulseUpdate: Boolean = false) {
         try {
             var packetStart = System.currentTimeMillis()

Server/worldprops/default.conf

@@ -22,7 +22,9 @@ noauth_default_admin = true #NOTE: If we are not using auth, this determines whe
 #------------------------------------------------------------------------------------------------------
 #The limit on how many different accounts a player can log into per day.
 daily_accounts_per_ip = 3
-watchdog_enabled = false
+watchdog_enabled = true
+connectivity_check_url = "https://google.com,https://2009scape.org"
+connectivity_timeout = 500
 
 [database]
 database_name = "global"