feat: incremental sync in data store

frontend/src2/data_store/DataStoreList.vue

@@ -1,73 +1,68 @@
 <script setup lang="ts">
-import { Breadcrumbs, ListView } from 'frappe-ui'
-import { PlusIcon, SearchIcon } from 'lucide-vue-next'
+import { Breadcrumbs, Button } from 'frappe-ui'
+import { ChevronRightIcon, PlusIcon, RefreshCwIcon, SearchIcon } from 'lucide-vue-next'
 import { computed, onMounted, ref } from 'vue'
 import { getDatabaseLogo } from '../data_source/data_source'
-import useDataStore, { DataStoreTable } from './data_store'
+import type { DatabaseType } from '../data_source/data_source.types'
+import useDataStore, { DataStoreTable, SyncLog } from './data_store'
 import ImportTableDialog from './ImportTableDialog.vue'
+import SyncDialog from './SyncDialog.vue'
+import SyncLogSummary from './SyncLogSummary.vue'
 import session from '../session'
 import { __ } from '../translation'
 
-onMounted(() => {
-	dataStore.getTables()
-})
-
-const showImportTableDialog = ref(false)
-
 const dataStore = useDataStore()
 const searchQuery = ref('')
+const showImportTableDialog = ref(false)
+const showSyncDialog = ref(false)
+const syncDialogTable = ref<DataStoreTable | null>(null)
+const expandedTable = ref<string | null>(null)
+const syncLogs = ref<Record<string, SyncLog | null>>({})
+
+onMounted(() => dataStore.getTables())
+
 const tables = computed(() => dataStore.tables['__all'] || [])
-const normalizedSearchQuery = computed(() => searchQuery.value.toLowerCase().trim())
 
 const filteredTables = computed(() => {
-	const allTables = tables.value
-	const query = normalizedSearchQuery.value
+	const query = searchQuery.value.toLowerCase().trim()
+	if (!query) return tables.value
+	return tables.value.filter((t: DataStoreTable) => t.table_name.toLowerCase().includes(query))
+})
 
-	if (!query) return allTables
+function tableKey(table: DataStoreTable) {
+	return `${table.data_source}:${table.table_name}`
+}
 
-	return allTables.filter((table: DataStoreTable) => {
-		return table.table_name.toLowerCase().includes(query)
-	})
-})
+function isExpanded(table: DataStoreTable) {
+	return expandedTable.value === tableKey(table)
+}
+
+function toggleExpand(table: DataStoreTable) {
+	const key = tableKey(table)
+	if (expandedTable.value === key) {
+		expandedTable.value = null
+		return
+	}
+	expandedTable.value = key
+	if (!(key in syncLogs.value)) {
+		dataStore.getLastSyncLog(table.data_source, table.table_name).then((log) => {
+			syncLogs.value[key] = log
+		})
+	}
+}
 
-const listOptions = computed(() => ({
-	columns: [
-		{
-			label: __('Table Name'),
-			key: 'table_name',
-		},
-		{
-			label: __('Data Source'),
-			key: 'data_source',
-			prefix: (props: any) => {
-				const table = props.row as DataStoreTable
-				return getDatabaseLogo(table.database_type, 'sm')
-			},
-		},
-		{
-			label: __('Last Synced'),
-			key: 'last_synced_from_now',
-		},
-	],
-	rows: filteredTables.value,
-	rowKey: 'name',
-	options: {
-		showTooltip: false,
-		emptyState: {
-			title: __('No Tables Stored'),
-			description: __('No tables found in the data store.'),
-			button: session.user.is_admin
-				? {
-						label: __('Import Table'),
-						iconLeft: 'plus',
-						variant: 'solid',
-						loading: false,
-						onClick: () => (showImportTableDialog.value = true),
-				  }
-				: undefined,
-		},
-	},
-}))
+function openSyncDialog(table: DataStoreTable) {
+	syncDialogTable.value = table
+	showSyncDialog.value = true
+}
+
+const logoCache = new Map()
+function getCachedLogo(dbType: DatabaseType) {
+	if (!logoCache.has(dbType)) {
+		logoCache.set(dbType, getDatabaseLogo(dbType, 'sm'))
+	}
+	return logoCache.get(dbType)
+}
 </script>
 
 <template>
@@ -87,16 +82,98 @@ const listOptions = computed(() => ({
 		</div>
 	</header>
 
-	<div class="mb-4 flex h-full flex-col gap-3 overflow-auto px-5 py-3">
+	<div class="flex h-full flex-col gap-3 overflow-auto px-5 py-3">
 		<div class="flex gap-2 overflow-visible py-1">
-			<FormControl :placeholder="__('Search')" v-model="searchQuery" :debounce="300">
+			<FormControl :placeholder="__('Search table')" v-model="searchQuery" :debounce="300">
 				<template #prefix>
 					<SearchIcon class="h-4 w-4 text-gray-500" />
 				</template>
 			</FormControl>
 		</div>
-		<ListView class="h-full" v-bind="listOptions"> </ListView>
+
+		<div
+			v-if="!filteredTables.length"
+			class="flex flex-1 flex-col items-center justify-center gap-2 text-gray-500"
+		>
+			<p class="text-sm">{{ __('No tables found') }}</p>
+			<Button
+				v-if="session.user.is_admin && !searchQuery"
+				:label="__('Import Table')"
+				variant="solid"
+				@click="showImportTableDialog = true"
+			>
+				<template #prefix>
+					<PlusIcon class="w-4" />
+				</template>
+			</Button>
+		</div>
+
+		<div v-else class="flex flex-col">
+			<div
+				class="flex items-center gap-3 rounded bg-gray-50 px-4 py-2 text-sm font-medium text-gray-600"
+			>
+				<div class="w-5"></div>
+				<div style="flex: 2">{{ __('Table') }}</div>
+				<div style="flex: 1; min-width: 140px">{{ __('Data Source') }}</div>
+				<div style="min-width: 120px">{{ __('Last Synced') }}</div>
+				<div class="w-8"></div>
+			</div>
+
+			<div class="flex flex-col divide-y">
+				<div v-for="table in filteredTables" :key="tableKey(table)" class="flex flex-col">
+					<div
+						class="flex cursor-pointer items-center gap-3 px-4 py-2.5 hover:bg-gray-50"
+						@click="toggleExpand(table)"
+					>
+						<ChevronRightIcon
+							class="h-4 w-4 flex-shrink-0 text-gray-400"
+							:class="{ 'rotate-90': isExpanded(table) }"
+							stroke-width="1.5"
+						/>
+
+						<div class="min-w-0" style="flex: 2">
+							<p class="truncate text-sm font-medium text-gray-900">
+								{{ table.table_name }}
+							</p>
+						</div>
+
+						<div
+							class="flex items-center gap-1.5 text-sm text-gray-600"
+							style="flex: 1; min-width: 140px"
+						>
+							<component :is="getCachedLogo(table.database_type)" />
+							<span class="truncate">{{ table.data_source }}</span>
+						</div>
+
+						<div class="text-sm text-gray-500" style="min-width: 120px">
+							{{ table.last_synced_from_now || __('Never') }}
+						</div>
+
+						<Button
+							v-if="session.user.is_admin"
+							variant="ghost"
+							size="sm"
+							@click.stop="openSyncDialog(table)"
+						>
+							<template #icon>
+								<RefreshCwIcon
+									class="h-3.5 w-3.5 text-gray-600"
+									stroke-width="1.5"
+								/>
+							</template>
+						</Button>
+					</div>
+
+					<SyncLogSummary
+						v-if="isExpanded(table)"
+						:log="syncLogs[tableKey(table)]"
+						:sync-mode="table.sync_mode"
+					/>
+				</div>
+			</div>
+		</div>
 	</div>
 
 	<ImportTableDialog v-model="showImportTableDialog" />
+	<SyncDialog v-model="showSyncDialog" :table="syncDialogTable" />
 </template>

frontend/src2/data_store/SyncDialog.vue

@@ -0,0 +1,109 @@
+<script setup lang="ts">
+import { ref, watch } from 'vue'
+import useDataStore, { DataStoreTable } from './data_store'
+import { __ } from '../translation'
+
+const show = defineModel<boolean>({ default: false })
+const props = defineProps<{ table: DataStoreTable | null }>()
+
+const dataStore = useDataStore()
+const isLoading = ref(false)
+const selectedMode = ref('Incremental Sync')
+
+watch(show, (val) => {
+	if (val) {
+		selectedMode.value = 'Incremental Sync'
+		isLoading.value = false
+	}
+})
+
+async function sync() {
+	if (!props.table) return
+	isLoading.value = true
+	try {
+		if (selectedMode.value === 'Incremental Sync') {
+			await dataStore.syncTable(props.table.data_source, props.table.table_name)
+		} else {
+			await dataStore.fullRefreshTable(props.table.data_source, props.table.table_name)
+		}
+		show.value = false
+		dataStore.getTables()
+	} finally {
+		isLoading.value = false
+	}
+}
+</script>
+
+<template>
+	<Dialog
+		:modelValue="show"
+		@update:modelValue="show = $event"
+		:options="{ title: __('Sync Table'), size: 'sm' }"
+	>
+		<template #body-content>
+			<div class="flex flex-col gap-3">
+				<p class="text-sm text-gray-600">
+					{{ __('Choose how to sync') }}
+					<span class="font-medium text-gray-800">{{ props.table?.table_name }}</span>
+				</p>
+
+				<div
+					class="flex cursor-pointer items-start gap-3 rounded-lg border p-3 transition-colors"
+					:class="
+						selectedMode === 'Incremental Sync'
+							? 'border-blue-500 bg-blue-50'
+							: 'hover:bg-gray-50'
+					"
+					@click="selectedMode = 'Incremental Sync'"
+				>
+					<div class="flex-1">
+						<p class="text-sm font-medium text-gray-900">
+							{{ __('Incremental Sync') }}
+						</p>
+						<p class="mt-0.5 text-xs text-gray-500">
+							{{
+								__(
+									'Only syncs new and modified rows. Faster and recommended for most cases.',
+								)
+							}}
+						</p>
+					</div>
+					<span class="flex-shrink-0 text-xs text-green-600">{{
+						__('Recommended')
+					}}</span>
+				</div>
+
+				<div
+					class="flex cursor-pointer items-start gap-3 rounded-lg border p-3 transition-colors"
+					:class="
+						selectedMode === 'Full Refresh'
+							? 'border-blue-500 bg-blue-50'
+							: 'hover:bg-gray-50'
+					"
+					@click="selectedMode = 'Full Refresh'"
+				>
+					<div class="flex-1">
+						<p class="text-sm font-medium text-gray-900">{{ __('Full Refresh') }}</p>
+						<p class="mt-0.5 text-xs text-gray-500">
+							{{
+								__(
+									'Replaces all data with a fresh copy from the source. Slower but ensures complete accuracy.',
+								)
+							}}
+						</p>
+					</div>
+				</div>
+
+				<div class="flex justify-end gap-2 pt-1">
+					<Button :label="__('Cancel')" variant="outline" @click="show = false" />
+					<Button
+						:label="__('Sync Now')"
+						variant="solid"
+						:loading="isLoading"
+						@click="sync"
+					/>
+				</div>
+			</div>
+		</template>
+	</Dialog>
+</template>