Avoid deletion of non-synced bills

app/src/main/java/net/helcel/cowspent/android/main/BillsListComponents.kt

@@ -3,6 +3,9 @@ package net.helcel.cowspent.android.main
 import androidx.compose.foundation.background
 import androidx.compose.foundation.clickable
 import androidx.compose.foundation.layout.Arrangement
+import androidx.compose.foundation.layout.ColumnScope
+import androidx.compose.foundation.lazy.LazyColumn
+import androidx.compose.material.CircularProgressIndicator
 import androidx.compose.foundation.layout.Box
 import androidx.compose.foundation.layout.Column
 import androidx.compose.foundation.layout.Row
@@ -69,13 +72,43 @@ fun EmptyProjectsState(onConfigureNextcloud: () -> Unit, onAddManually: () -> Un
     }
 }
 
+/**
+ * A full-height, centred state that a pull to refresh can still act on.
+ *
+ * PullRefresh reads the gesture from a nested scroll source, and a plain Column offers none - so
+ * on an empty list, which is exactly when a refresh is wanted most, the pull never fires. A
+ * single-item LazyColumn has nowhere to scroll but does provide that source.
+ */
+@Composable
+private fun RefreshableFullScreenState(content: @Composable ColumnScope.() -> Unit) {
+    LazyColumn(modifier = Modifier.fillMaxSize()) {
+        item {
+            Column(
+                modifier = Modifier.fillParentMaxSize().padding(16.dp),
+                verticalArrangement = Arrangement.Center,
+                horizontalAlignment = Alignment.CenterHorizontally,
+                content = content
+            )
+        }
+    }
+}
+
+@Composable
+fun LoadingBillsState() {
+    RefreshableFullScreenState {
+        CircularProgressIndicator()
+        Spacer(modifier = Modifier.height(16.dp))
+        Text(
+            text = stringResource(R.string.error_loading),
+            style = MaterialTheme.typography.subtitle1,
+            color = MaterialTheme.colors.onSurface.copy(alpha = 0.6f)
+        )
+    }
+}
+
 @Composable
 fun EmptyMembersState() {
-    Column(
+    RefreshableFullScreenState {
-        modifier = Modifier.fillMaxSize().padding(16.dp),
-        verticalArrangement = Arrangement.Center,
-        horizontalAlignment = Alignment.CenterHorizontally
-    ) {
         Text(
             text = stringResource(R.string.error_no_members).uppercase(), 
             style = MaterialTheme.typography.subtitle1, 
@@ -94,11 +127,7 @@ fun EmptyMembersState() {
 
 @Composable
 fun EmptyBillsState() {
-    Column(
+    RefreshableFullScreenState {
-        modifier = Modifier.fillMaxSize().padding(16.dp),
-        verticalArrangement = Arrangement.Center,
-        horizontalAlignment = Alignment.CenterHorizontally
-    ) {
         Text(
             text = stringResource(R.string.error_no_bills).uppercase(), 
             style = MaterialTheme.typography.subtitle1, 
@@ -203,11 +232,7 @@ fun SectionHeader(title: String) {
 
 @Composable
 fun EmptyState() {
-    Column(
+    RefreshableFullScreenState {
-        modifier = Modifier.fillMaxSize(),
-        verticalArrangement = Arrangement.Center,
-        horizontalAlignment = Alignment.CenterHorizontally
-    ) {
         Text(
             text = stringResource(R.string.error_no_bills).uppercase(), 
             style = MaterialTheme.typography.subtitle1, 

app/src/main/java/net/helcel/cowspent/android/main/BillsListScreen.kt

@@ -409,6 +409,10 @@ fun BillsListScreen(
             when {
                 viewModel.showNoProjects -> EmptyProjectsState(onAccountSwitcherClick, onAddProjectClick)
                 viewModel.showNoMembers -> EmptyMembersState()
+                // A large project takes a long time on its first sync. Until that finishes there
+                // is nothing stored for it yet, and reporting that as "no bills" tells the user
+                // the project is empty when it is still downloading.
+                viewModel.isRefreshing && viewModel.bills.isEmpty() -> LoadingBillsState()
                 viewModel.showNoBills -> EmptyBillsState()
                 viewModel.bills.isEmpty() -> EmptyState()
                 else -> {

app/src/main/java/net/helcel/cowspent/android/main/BillsListViewActivity.kt

@@ -94,14 +94,16 @@ class BillsListViewActivity :
     private val syncCallBack = object : ICallback {
         override fun onFinish() {
             mActionMode?.finish()
-            refreshLists()
             viewModel.isRefreshing = false
+            refreshLists()
         }
 
         override fun onFinish(result: String, message: String) {}
 
         override fun onScheduled() {
-            viewModel.isRefreshing = false
+            // Being queued is not being done. Clearing the indicator here stops the spinner while a
+            // large project is still downloading; synchronize() already releases it when nothing
+            // actually started.
         }
     }
 
@@ -788,11 +790,23 @@ class BillsListViewActivity :
         lifecycleScope.launch {
             val remoteProjects = withContext(Dispatchers.IO) { db.projects }
                 .filter { !it.isLocal && !it.isArchived }
+                // Only the first one scheduled actually starts - the rest queue behind it - and it
+                // takes the screen's callback with it. So the selected project goes first: otherwise
+                // the spinner follows a project the user is not looking at, stops when that one
+                // finishes, and theirs is reported as empty while it is still waiting its turn.
+                .sortedByDescending { it.id == selectedProjectId }
 
             viewModel.isRefreshing = true
             db.cowspentServerSyncHelper.addCallbackPull(syncCallBack)
 
             val started = if (accountSyncDue) {
+                // The pass is throttled by this stamp, so it has to be written here rather than
+                // left to the account sync: that only runs when an account is configured and only
+                // records itself on success, so without one the stamp stayed at zero and the pass
+                // repeated on every single resume, re-scheduling every project each time.
+                preferences.edit {
+                    putLong(getString(R.string.pref_key_last_account_sync_timestamp), now)
+                }
                 if (CowspentServerSyncHelper.isNextcloudAccountConfigured(applicationContext)) {
                     db.cowspentServerSyncHelper.runAccountProjectsSync()
                 }
@@ -804,14 +818,24 @@ class BillsListViewActivity :
             } else {
                 val selectedProj = remoteProjects.find { it.id == selectedProjectId }
                 val lastSync = preferences.getLong(lastProjectSyncKey(selectedProjectId), 0L)
-                val due = trigger == SyncTrigger.MANUAL ||
+                // The pull writes the project cursor only after it has applied everything, so a cursor
+                // still at zero means no sync ever finished and nothing is stored locally.
+                // Throttling that strands the user on an empty list with no spinner - which is what
+                // the back gesture does to a large project, since it destroys the activity while
+                // the sync is still running. IHateMoney never advances the cursor, so the interval
+                // stays in charge there.
+                val neverCompleted = selectedProj != null &&
+                    selectedProj.type != ProjectType.IHATEMONEY &&
+                    (selectedProj.lastSyncedTimestamp ?: 0L) == 0L
+                val due = trigger == SyncTrigger.MANUAL || neverCompleted ||
                     now - lastSync > SELECTED_PROJECT_SYNC_INTERVAL_MS
+                val fullSync = trigger == SyncTrigger.MANUAL &&
+                    !partialRefreshPreferred(preferences, selectedProjectId, now)
                 if (selectedProj != null && due &&
-                    db.cowspentServerSyncHelper.scheduleSync(
+                    db.cowspentServerSyncHelper.scheduleSync(false, selectedProj, fullSync) != null
-                        false, selectedProj, trigger == SyncTrigger.MANUAL
-                    ) != null
                 ) {
                     markProjectSynced(preferences, selectedProj.id, now)
+                    if (fullSync) markFullySynced(preferences, selectedProj.id, now)
                     1
                 } else 0
             }
@@ -823,6 +847,30 @@ class BillsListViewActivity :
     }
 
     private fun lastProjectSyncKey(projectId: Long) = "lastProjectSyncTimestamp_$projectId"
+    private fun lastFullSyncKey(projectId: Long) = "lastFullSyncTimestamp_$projectId"
+
+    private fun markFullySynced(preferences: SharedPreferences, projectId: Long, at: Long) {
+        preferences.edit { putLong(lastFullSyncKey(projectId), at) }
+    }
+
+    /**
+     * Whether a pull to refresh should settle for the cheaper paged walk.
+     *
+     * A refresh by hand normally re-downloads the whole project, which is what makes it a usable
+     * repair when the local copy looks wrong. On a large project that is expensive to repeat, so
+     * with beta features on a refresh that follows a recent full one walks the recent pages
+     * instead - the common case of pulling twice in a row costs a page rather than the project.
+     */
+    private fun partialRefreshPreferred(
+        preferences: SharedPreferences,
+        projectId: Long,
+        now: Long
+    ): Boolean {
+        if (!preferences.getBoolean(getString(R.string.pref_key_beta_features), false)) return false
+        val lastFull = preferences.getLong(lastFullSyncKey(projectId), 0L)
+        if (lastFull == 0L) return false
+        return now - lastFull < SyncSettings.intervalMinutes(applicationContext) * 60 * 1000L
+    }
     private fun markProjectSynced(preferences: SharedPreferences, projectId: Long, at: Long) {
         preferences.edit { putLong(lastProjectSyncKey(projectId), at) }
     }

app/src/main/java/net/helcel/cowspent/persistence/CowspentSQLiteOpenHelper.kt

@@ -448,6 +448,26 @@ class CowspentSQLiteOpenHelper private constructor(val context: Context) :
 
     // --- Bills logic ---
 
+    /**
+     * Runs [block] as one database transaction.
+     *
+     * Every write here commits on its own otherwise, which for a sync means a committed
+     * transaction per bill and another per ower - the dominant cost of a large project's first
+     * sync. Callers batch in chunks rather than wrapping everything: a sync interrupted halfway
+     * then keeps the chunks it already committed instead of rolling the lot back.
+     */
+    fun <T> inTransaction(block: () -> T): T {
+        val db = writableDatabase
+        db.beginTransaction()
+        try {
+            val result = block()
+            db.setTransactionSuccessful()
+            return result
+        } finally {
+            db.endTransaction()
+        }
+    }
+
     fun addBill(b: DBBill): Long {
         val db = writableDatabase
         val values = ContentValues()
@@ -627,11 +647,25 @@ class CowspentSQLiteOpenHelper private constructor(val context: Context) :
         val cursor = db.query(table_bills, columnsBills, selection, selectionArgs, null, null, orderBy)
         val bills: MutableList<DBBill> = ArrayList()
         while (cursor.moveToNext()) {
-            val bill = getBillFromCursor(cursor)
+            bills.add(getBillFromCursor(cursor))
-            bill.billOwers = getBillowersOfBill(bill.id)
-            bills.add(bill)
         }
         cursor.close()
+        if (bills.isEmpty()) return bills
+
+        // Every ower of the matched bills in one query, keyed back to its bill.
+        //
+        // Fetching them per bill meant a project with thousands of bills issued thousands of
+        // queries on every list refresh - and that refresh runs on resume, on switching project,
+        // and after each sync. The bill ids are re-selected as a subquery rather than bound as
+        // arguments, which would blow past SQLite's limit on bound variables for a large project.
+        val owersByBill = getBillOwersCustom(
+            "$key_billId IN (SELECT $key_id FROM $table_bills WHERE $selection)",
+            selectionArgs,
+            null
+        ).groupBy { it.billId }
+        for (bill in bills) {
+            bill.billOwers = owersByBill[bill.id].orEmpty()
+        }
         return bills
     }
 

app/src/main/java/net/helcel/cowspent/persistence/CowspentServerSyncHelper.kt

@@ -50,7 +50,6 @@ class CowspentServerSyncHelper private constructor(private val dbHelper: Cowspen
     private var syncActive = false
     private var syncAccountProjectsActive = false
 
-    private var callbacksPush: MutableList<ICallback> = ArrayList()
     private var callbacksPull: MutableList<ICallback> = ArrayList()
 
     init {
@@ -101,8 +100,6 @@ class CowspentServerSyncHelper private constructor(private val dbHelper: Cowspen
             if (project != null) {
                 Log.d(TAG, "... starting now")
                 val syncTask = SyncTask(onlyLocalChanges, project, forceFullSync)
-                syncTask.addCallbacks(callbacksPush)
-                callbacksPush = ArrayList()
                 if (!onlyLocalChanges) {
                     syncTask.addCallbacks(callbacksPull)
                     callbacksPull = ArrayList()
@@ -114,14 +111,8 @@ class CowspentServerSyncHelper private constructor(private val dbHelper: Cowspen
         } else if (!onlyLocalChanges) {
             Log.d(TAG, "... scheduled")
             projectIdsToSync.add(projId)
-            for (callback in callbacksPush) {
-                callback.onScheduled()
-            }
         } else {
             Log.d(TAG, "... do nothing")
-            for (callback in callbacksPush) {
-                callback.onScheduled()
-            }
         }
         return null
     }
@@ -808,7 +799,9 @@ class CowspentServerSyncHelper private constructor(private val dbHelper: Cowspen
             if (usePagedWalk) {
                 walkBillPages(idMaps, localBillsByRemoteId)?.let { return it }
             }
-            return fetchAllBills(idMaps)
+            // A project with nothing stored has no cursor worth sending, so it asks for everything.
+            val since = if (forceFullSync || localBillsByRemoteId.isEmpty()) 0L else null
+            return fetchAllBills(idMaps, since)
         }
 
         /**
@@ -883,9 +876,14 @@ class CowspentServerSyncHelper private constructor(private val dbHelper: Cowspen
             return RemoteBills(bills, emptyList(), syncTimestamp)
         }
 
-        private fun fetchAllBills(idMaps: RemoteIdMaps): RemoteBills {
+        /**
-            Log.d(TAG, "Starting full sync for project ${project.remoteId}")
+         * One request for the whole collection. [since] 0 asks for everything; the project cursor
-            val response = client!!.getBills(project)
+         * asks only for what changed. Either way the response carries the full id list, which is
+         * what lets vanished bills be removed locally.
+         */
+        private fun fetchAllBills(idMaps: RemoteIdMaps, since: Long?): RemoteBills {
+            Log.d(TAG, "Fetching bills for ${project.remoteId} (since=${since ?: project.lastSyncedTimestamp})")
+            val response = client!!.getBills(project, since = since)
             return if (project.type == ProjectType.IHATEMONEY) {
                 val bills = response.getBillsIHM(
                     project.id, idMaps.members, idMaps.categories, idMaps.paymentModes
@@ -905,6 +903,18 @@ class CowspentServerSyncHelper private constructor(private val dbHelper: Cowspen
         private fun applyRemoteBills(
             remoteBills: List<DBBill>,
             localBillsByRemoteId: Map<Long, DBBill>
+        ) {
+            // Committed in chunks: one transaction per chunk turns a per-row commit into one write
+            // for the whole batch, while still leaving finished chunks on disk if the sync is cut
+            // short - a project part-way through is far better than one that rolled back.
+            remoteBills.chunked(BILL_APPLY_CHUNK).forEach { chunk ->
+                dbHelper.inTransaction { applyBillChunk(chunk, localBillsByRemoteId) }
+            }
+        }
+
+        private fun applyBillChunk(
+            remoteBills: List<DBBill>,
+            localBillsByRemoteId: Map<Long, DBBill>
         ) {
             for (remoteBill in remoteBills) {
                 val localBill = localBillsByRemoteId[remoteBill.remoteId]
@@ -966,6 +976,10 @@ class CowspentServerSyncHelper private constructor(private val dbHelper: Cowspen
                 }
 
             for (localBill in localBills) {
+                // A bill still waiting to be pushed has no remote id yet, so it is absent from the
+                // server list for the ordinary reason that the server has never seen it. Deleting
+                // it here would throw away a bill the user added while the sync was running.
+                if (localBill.state != DBBill.STATE_OK) continue
                 if (localBill.remoteId !in stillRemote) {
                     dbHelper.deleteBill(localBill.id)
                     nbPulledDeletedBills++
@@ -1801,6 +1815,9 @@ class CowspentServerSyncHelper private constructor(private val dbHelper: Cowspen
          */
         private const val UNCHANGED_RUN_TO_SETTLE = 25
 
+        /** Bills written per transaction while applying a pull. */
+        private const val BILL_APPLY_CHUNK = 500
+
         private var instance: CowspentServerSyncHelper? = null
         private val projectIdsToSync: MutableList<Long> = ArrayList()
 

app/src/main/java/net/helcel/cowspent/util/VersatileProjectSyncClient.kt

@@ -689,12 +689,18 @@ class VersatileProjectSyncClient(
     }
 
     @Throws(JSONException::class, IOException::class, TokenMismatchException::class, NextcloudHttpRequestFailedException::class)
+    /**
+     * [since] overrides the project cursor sent as lastChanged. Passing 0 asks the server for the
+     * whole project rather than the changes since the last sync; omitting it keeps the cursor, so
+     * the server returns only what has moved.
+     */
     fun getBills(
         project: DBProject,
         offset: Int? = null,
         limit: Int? = null,
         reverse: Boolean? = null,
-        deleted: Int? = null
+        deleted: Int? = null,
+        since: Long? = null
     ): ServerResponse.BillsResponse {
         var target: String
         var username: String?
@@ -706,7 +712,7 @@ class VersatileProjectSyncClient(
         val paramValues: MutableList<String> = ArrayList()
 
         if (ProjectType.COSPEND == project.type) {
-            val tsLastSync = project.lastSyncedTimestamp
+            val tsLastSync = since ?: project.lastSyncedTimestamp
             if (offset == null) {
                 if (cospendVersionGT161) {
                     paramKeys.add("lastChanged")