From 8b68869912162b1831bbbcb42aec3c9555dc43fa Mon Sep 17 00:00:00 2001 From: Ivan Zinchenko Date: Fri, 13 Feb 2026 22:01:35 +0300 Subject: [PATCH 1/7] WIP --- lab/vtpc/lib/vtpc.c | 714 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 708 insertions(+), 6 deletions(-) diff --git a/lab/vtpc/lib/vtpc.c b/lab/vtpc/lib/vtpc.c index 173ab1b..1940500 100644 --- a/lab/vtpc/lib/vtpc.c +++ b/lab/vtpc/lib/vtpc.c @@ -1,30 +1,732 @@ #include "vtpc.h" +#include #include +#include #include +#include +#include +#include +#include #include #include +#define VTPC_MAX_OPEN_FILES 256 +#define VTPC_MAX_CACHE_PAGES 128 +#define VTPC_MAX_A1OUT_PAGES 128 + +enum vtpc_resident_list { + VTPC_LIST_NONE = 0, + VTPC_LIST_A1IN = 1, + VTPC_LIST_AM = 2, +}; + +typedef struct { + int head; + int tail; + size_t size; +} vtpc_list; + +typedef struct { + int used; + int prev; + int next; + int list; + int dirty; + int file_slot; + off_t page_no; + unsigned char* data; +} vtpc_resident_page; + +typedef struct { + int used; + int prev; + int next; + int file_slot; + off_t page_no; +} vtpc_ghost_page; + +typedef struct { + int used; + int os_fd; + int open_flags; + off_t pos; + off_t size; + int size_dirty; +} vtpc_file; + +typedef struct { + int initialized; + size_t page_size; + size_t max_resident; + size_t max_a1out; + size_t kin; + + vtpc_list a1in; + vtpc_list am; + vtpc_list a1out; + + vtpc_file files[VTPC_MAX_OPEN_FILES]; + vtpc_resident_page resident[VTPC_MAX_CACHE_PAGES]; + vtpc_ghost_page ghosts[VTPC_MAX_A1OUT_PAGES]; +} vtpc_state; + +static vtpc_state g_state = {0}; + +static size_t vtpc_min_size(size_t lhs, size_t rhs) { + return lhs < rhs ? lhs : rhs; +} + +static int vtpc_open_fallback(const char* path, int mode, int access) { + int fd = open(path, mode, access); + return fd; +} + +static int vtpc_state_init(void) { + if (g_state.initialized) { + return 0; + } + + long page_sz = sysconf(_SC_PAGESIZE); + if (page_sz <= 0) { + page_sz = 4096; + } + + g_state.page_size = (size_t)page_sz; + g_state.max_resident = VTPC_MAX_CACHE_PAGES; + g_state.max_a1out = VTPC_MAX_A1OUT_PAGES; + g_state.kin = g_state.max_resident / 4; + if (g_state.kin == 0) { + g_state.kin = 1; + } + + g_state.a1in.head = -1; + g_state.a1in.tail = -1; + g_state.am.head = -1; + g_state.am.tail = -1; + g_state.a1out.head = -1; + g_state.a1out.tail = -1; + + g_state.initialized = 1; + return 0; +} + +static void vtpc_resident_list_push_back(vtpc_list* list, int idx) { + vtpc_resident_page* page = &g_state.resident[idx]; + page->prev = list->tail; + page->next = -1; + if (list->tail != -1) { + g_state.resident[list->tail].next = idx; + } else { + list->head = idx; + } + list->tail = idx; + ++list->size; +} + +static void vtpc_resident_list_remove(vtpc_list* list, int idx) { + vtpc_resident_page* page = &g_state.resident[idx]; + if (page->prev != -1) { + g_state.resident[page->prev].next = page->next; + } else { + list->head = page->next; + } + if (page->next != -1) { + g_state.resident[page->next].prev = page->prev; + } else { + list->tail = page->prev; + } + page->prev = -1; + page->next = -1; + if (list->size > 0) { + --list->size; + } +} + +static int vtpc_resident_list_pop_front(vtpc_list* list) { + if (list->head == -1) { + return -1; + } + int idx = list->head; + vtpc_resident_list_remove(list, idx); + return idx; +} + +static void vtpc_ghost_list_push_back(vtpc_list* list, int idx) { + vtpc_ghost_page* page = &g_state.ghosts[idx]; + page->prev = list->tail; + page->next = -1; + if (list->tail != -1) { + g_state.ghosts[list->tail].next = idx; + } else { + list->head = idx; + } + list->tail = idx; + ++list->size; +} + +static void vtpc_ghost_list_remove(vtpc_list* list, int idx) { + vtpc_ghost_page* page = &g_state.ghosts[idx]; + if (page->prev != -1) { + g_state.ghosts[page->prev].next = page->next; + } else { + list->head = page->next; + } + if (page->next != -1) { + g_state.ghosts[page->next].prev = page->prev; + } else { + list->tail = page->prev; + } + page->prev = -1; + page->next = -1; + if (list->size > 0) { + --list->size; + } +} + +static int vtpc_ghost_list_pop_front(vtpc_list* list) { + if (list->head == -1) { + return -1; + } + int idx = list->head; + vtpc_ghost_list_remove(list, idx); + return idx; +} + +static vtpc_file* vtpc_get_file(int fd) { + if (fd < 0 || fd >= VTPC_MAX_OPEN_FILES || !g_state.files[fd].used) { + errno = EBADF; + return NULL; + } + return &g_state.files[fd]; +} + +static int vtpc_find_resident_page(int file_slot, off_t page_no) { + for (int i = 0; i < VTPC_MAX_CACHE_PAGES; ++i) { + vtpc_resident_page* page = &g_state.resident[i]; + if (page->used && page->file_slot == file_slot && page->page_no == page_no) { + return i; + } + } + return -1; +} + +static int vtpc_find_ghost_page(int file_slot, off_t page_no) { + for (int i = 0; i < VTPC_MAX_A1OUT_PAGES; ++i) { + vtpc_ghost_page* page = &g_state.ghosts[i]; + if (page->used && page->file_slot == file_slot && page->page_no == page_no) { + return i; + } + } + return -1; +} + +static int vtpc_alloc_resident_slot(void) { + for (int i = 0; i < VTPC_MAX_CACHE_PAGES; ++i) { + vtpc_resident_page* page = &g_state.resident[i]; + if (!page->used) { + if (page->data == NULL) { + if (posix_memalign((void**)&page->data, g_state.page_size, g_state.page_size) != 0) { + errno = ENOMEM; + return -1; + } + } + page->used = 1; + page->prev = -1; + page->next = -1; + page->list = VTPC_LIST_NONE; + page->dirty = 0; + return i; + } + } + errno = ENOMEM; + return -1; +} + +static int vtpc_alloc_ghost_slot(void) { + for (int i = 0; i < VTPC_MAX_A1OUT_PAGES; ++i) { + if (!g_state.ghosts[i].used) { + g_state.ghosts[i].used = 1; + g_state.ghosts[i].prev = -1; + g_state.ghosts[i].next = -1; + return i; + } + } + errno = ENOMEM; + return -1; +} + +static int vtpc_pread_page(int os_fd, off_t page_no, unsigned char* data) { + const off_t offset = (off_t)(page_no * (off_t)g_state.page_size); + size_t done = 0; + while (done < g_state.page_size) { + ssize_t rc = pread(os_fd, data + done, g_state.page_size - done, offset + (off_t)done); + if (rc < 0) { + if (errno == EINTR) { + continue; + } + return -1; + } + if (rc == 0) { + memset(data + done, 0, g_state.page_size - done); + break; + } + done += (size_t)rc; + } + return 0; +} + +static int vtpc_pwrite_page(int os_fd, off_t page_no, const unsigned char* data) { + const off_t offset = (off_t)(page_no * (off_t)g_state.page_size); + size_t done = 0; + while (done < g_state.page_size) { + ssize_t rc = pwrite(os_fd, data + done, g_state.page_size - done, offset + (off_t)done); + if (rc < 0) { + if (errno == EINTR) { + continue; + } + return -1; + } + if (rc == 0) { + errno = EIO; + return -1; + } + done += (size_t)rc; + } + return 0; +} + +static int vtpc_flush_page(int idx) { + vtpc_resident_page* page = &g_state.resident[idx]; + if (!page->used || !page->dirty) { + return 0; + } + + vtpc_file* file = &g_state.files[page->file_slot]; + if (!file->used) { + errno = EBADF; + return -1; + } + + if (vtpc_pwrite_page(file->os_fd, page->page_no, page->data) < 0) { + return -1; + } + page->dirty = 0; + return 0; +} + +static void vtpc_trim_a1out(void) { + while (g_state.a1out.size > g_state.max_a1out) { + int idx = vtpc_ghost_list_pop_front(&g_state.a1out); + if (idx == -1) { + break; + } + g_state.ghosts[idx].used = 0; + } +} + +static int vtpc_evict_one(void) { + int victim = -1; + int victim_from_a1in = 0; + + if (g_state.a1in.size > g_state.kin) { + victim = vtpc_resident_list_pop_front(&g_state.a1in); + victim_from_a1in = 1; + } else if (g_state.am.size > 0) { + victim = vtpc_resident_list_pop_front(&g_state.am); + } else { + victim = vtpc_resident_list_pop_front(&g_state.a1in); + victim_from_a1in = 1; + } + + if (victim == -1) { + errno = ENOMEM; + return -1; + } + + vtpc_resident_page* page = &g_state.resident[victim]; + page->list = VTPC_LIST_NONE; + + if (vtpc_flush_page(victim) < 0) { + return -1; + } + + if (victim_from_a1in) { + int ghost_idx = vtpc_alloc_ghost_slot(); + if (ghost_idx >= 0) { + g_state.ghosts[ghost_idx].file_slot = page->file_slot; + g_state.ghosts[ghost_idx].page_no = page->page_no; + vtpc_ghost_list_push_back(&g_state.a1out, ghost_idx); + vtpc_trim_a1out(); + } + } + + page->used = 0; + page->dirty = 0; + page->file_slot = -1; + page->page_no = 0; + return 0; +} + +static int vtpc_ensure_resident_capacity(void) { + while (g_state.a1in.size + g_state.am.size >= g_state.max_resident) { + if (vtpc_evict_one() < 0) { + return -1; + } + } + return 0; +} + +static void vtpc_move_resident_to_am(int idx) { + vtpc_resident_page* page = &g_state.resident[idx]; + if (page->list == VTPC_LIST_A1IN) { + vtpc_resident_list_remove(&g_state.a1in, idx); + } else if (page->list == VTPC_LIST_AM) { + vtpc_resident_list_remove(&g_state.am, idx); + } + page->list = VTPC_LIST_AM; + vtpc_resident_list_push_back(&g_state.am, idx); +} + +static int vtpc_get_or_load_page(int file_slot, off_t page_no, int need_existing, int* out_idx) { + int resident_idx = vtpc_find_resident_page(file_slot, page_no); + if (resident_idx >= 0) { + if (g_state.resident[resident_idx].list == VTPC_LIST_A1IN) { + vtpc_move_resident_to_am(resident_idx); + } else if (g_state.resident[resident_idx].list == VTPC_LIST_AM) { + vtpc_resident_list_remove(&g_state.am, resident_idx); + vtpc_resident_list_push_back(&g_state.am, resident_idx); + } + *out_idx = resident_idx; + return 0; + } + + if (vtpc_ensure_resident_capacity() < 0) { + return -1; + } + + int ghost_idx = vtpc_find_ghost_page(file_slot, page_no); + int idx = vtpc_alloc_resident_slot(); + if (idx < 0) { + return -1; + } + + vtpc_resident_page* page = &g_state.resident[idx]; + page->file_slot = file_slot; + page->page_no = page_no; + page->dirty = 0; + + if (need_existing) { + vtpc_file* file = &g_state.files[file_slot]; + if (vtpc_pread_page(file->os_fd, page_no, page->data) < 0) { + page->used = 0; + return -1; + } + } else { + memset(page->data, 0, g_state.page_size); + } + + if (ghost_idx >= 0) { + vtpc_ghost_list_remove(&g_state.a1out, ghost_idx); + g_state.ghosts[ghost_idx].used = 0; + page->list = VTPC_LIST_AM; + vtpc_resident_list_push_back(&g_state.am, idx); + } else { + page->list = VTPC_LIST_A1IN; + vtpc_resident_list_push_back(&g_state.a1in, idx); + } + + *out_idx = idx; + return 0; +} + +static int vtpc_flush_file_pages(vtpc_file* file, int file_slot, int with_fsync) { + for (int i = 0; i < VTPC_MAX_CACHE_PAGES; ++i) { + vtpc_resident_page* page = &g_state.resident[i]; + if (page->used && page->file_slot == file_slot && page->dirty) { + if (vtpc_flush_page(i) < 0) { + return -1; + } + } + } + + if (file->size_dirty) { + if (ftruncate(file->os_fd, file->size) < 0) { + return -1; + } + } + + if (with_fsync) { + if (fsync(file->os_fd) < 0) { + return -1; + } + } + + file->size_dirty = 0; + return 0; +} + +static void vtpc_drop_file_pages(int file_slot) { + for (int i = 0; i < VTPC_MAX_CACHE_PAGES; ++i) { + vtpc_resident_page* page = &g_state.resident[i]; + if (!page->used || page->file_slot != file_slot) { + continue; + } + + if (page->list == VTPC_LIST_A1IN) { + vtpc_resident_list_remove(&g_state.a1in, i); + } else if (page->list == VTPC_LIST_AM) { + vtpc_resident_list_remove(&g_state.am, i); + } + + page->used = 0; + page->dirty = 0; + page->list = VTPC_LIST_NONE; + page->file_slot = -1; + page->page_no = 0; + } + + for (int i = 0; i < VTPC_MAX_A1OUT_PAGES; ++i) { + vtpc_ghost_page* ghost = &g_state.ghosts[i]; + if (!ghost->used || ghost->file_slot != file_slot) { + continue; + } + vtpc_ghost_list_remove(&g_state.a1out, i); + ghost->used = 0; + ghost->file_slot = -1; + ghost->page_no = 0; + } +} + int vtpc_open(const char* path, int mode, int access) { - return open(path, mode, access); + if (vtpc_state_init() < 0) { + return -1; + } + + int open_mode = mode; +#ifdef O_DIRECT + open_mode |= O_DIRECT; +#endif + + int os_fd = open(path, open_mode, access); +#ifdef O_DIRECT + if (os_fd < 0 && (errno == EINVAL || errno == EOPNOTSUPP || errno == ENOTSUP)) { + os_fd = vtpc_open_fallback(path, mode, access); + } +#endif + if (os_fd < 0) { + return -1; + } + + int slot = -1; + for (int i = 0; i < VTPC_MAX_OPEN_FILES; ++i) { + if (!g_state.files[i].used) { + slot = i; + break; + } + } + if (slot < 0) { + (void)close(os_fd); + errno = EMFILE; + return -1; + } + + struct stat st; + if (fstat(os_fd, &st) < 0) { + int saved = errno; + (void)close(os_fd); + errno = saved; + return -1; + } + + g_state.files[slot].used = 1; + g_state.files[slot].os_fd = os_fd; + g_state.files[slot].open_flags = mode; + g_state.files[slot].pos = 0; + g_state.files[slot].size = st.st_size; + g_state.files[slot].size_dirty = 0; + + return slot; } int vtpc_close(int fd) { - return close(fd); + if (vtpc_state_init() < 0) { + return -1; + } + + vtpc_file* file = vtpc_get_file(fd); + if (file == NULL) { + return -1; + } + + int status = 0; + int saved_errno = 0; + if (vtpc_flush_file_pages(file, fd, 1) < 0) { + status = -1; + saved_errno = errno; + } + + vtpc_drop_file_pages(fd); + int os_fd = file->os_fd; + file->used = 0; + + if (close(os_fd) < 0 && status == 0) { + status = -1; + saved_errno = errno; + } + + if (status < 0) { + errno = saved_errno; + } + return status; } ssize_t vtpc_read(int fd, void* buf, size_t count) { - return read(fd, buf, count); + if (vtpc_state_init() < 0) { + return -1; + } + + vtpc_file* file = vtpc_get_file(fd); + if (file == NULL) { + return -1; + } + + int acc_mode = file->open_flags & O_ACCMODE; + if (!(acc_mode == O_RDONLY || acc_mode == O_RDWR)) { + errno = EBADF; + return -1; + } + + if (count == 0) { + return 0; + } + + if (file->pos >= file->size) { + return 0; + } + + size_t total = 0; + unsigned char* out = (unsigned char*)buf; + while (total < count && file->pos < file->size) { + off_t page_no = file->pos / (off_t)g_state.page_size; + size_t page_off = (size_t)(file->pos % (off_t)g_state.page_size); + size_t chunk = vtpc_min_size(count - total, g_state.page_size - page_off); + off_t available = file->size - file->pos; + if ((off_t)chunk > available) { + chunk = (size_t)available; + } + + int page_idx = -1; + if (vtpc_get_or_load_page(fd, page_no, 1, &page_idx) < 0) { + if (total > 0) { + return (ssize_t)total; + } + return -1; + } + + memcpy(out + total, g_state.resident[page_idx].data + page_off, chunk); + total += chunk; + file->pos += (off_t)chunk; + } + + return (ssize_t)total; } ssize_t vtpc_write(int fd, const void* buf, size_t count) { - return write(fd, buf, count); + if (vtpc_state_init() < 0) { + return -1; + } + + vtpc_file* file = vtpc_get_file(fd); + if (file == NULL) { + return -1; + } + + int acc_mode = file->open_flags & O_ACCMODE; + if (!(acc_mode == O_WRONLY || acc_mode == O_RDWR)) { + errno = EBADF; + return -1; + } + + if (count == 0) { + return 0; + } + + if ((file->open_flags & O_APPEND) != 0) { + file->pos = file->size; + } + + size_t total = 0; + const unsigned char* in = (const unsigned char*)buf; + while (total < count) { + off_t page_no = file->pos / (off_t)g_state.page_size; + size_t page_off = (size_t)(file->pos % (off_t)g_state.page_size); + size_t chunk = vtpc_min_size(count - total, g_state.page_size - page_off); + + int need_existing = !(page_off == 0 && chunk == g_state.page_size); + int page_idx = -1; + if (vtpc_get_or_load_page(fd, page_no, need_existing, &page_idx) < 0) { + if (total > 0) { + return (ssize_t)total; + } + return -1; + } + + memcpy(g_state.resident[page_idx].data + page_off, in + total, chunk); + g_state.resident[page_idx].dirty = 1; + + total += chunk; + file->pos += (off_t)chunk; + if (file->pos > file->size) { + file->size = file->pos; + file->size_dirty = 1; + } + } + + return (ssize_t)total; } off_t vtpc_lseek(int fd, off_t offset, int whence) { - return lseek(fd, offset, whence); + if (vtpc_state_init() < 0) { + return -1; + } + + vtpc_file* file = vtpc_get_file(fd); + if (file == NULL) { + return -1; + } + + off_t base = 0; + if (whence == SEEK_SET) { + base = 0; + } else if (whence == SEEK_CUR) { + base = file->pos; + } else if (whence == SEEK_END) { + base = file->size; + } else { + errno = EINVAL; + return -1; + } + + off_t next = base + offset; + if (next < 0) { + errno = EINVAL; + return -1; + } + + file->pos = next; + return next; } int vtpc_fsync(int fd) { - return fsync(fd); + if (vtpc_state_init() < 0) { + return -1; + } + + vtpc_file* file = vtpc_get_file(fd); + if (file == NULL) { + return -1; + } + + return vtpc_flush_file_pages(file, fd, 1); } From a54eb2560a2e35a7d8a330b57e50ec0da0324bfa Mon Sep 17 00:00:00 2001 From: Ivan Zinchenko Date: Fri, 13 Feb 2026 22:02:18 +0300 Subject: [PATCH 2/7] WIP --- .github/workflows/{vtsh.yml => vtpc.yml} | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) rename .github/workflows/{vtsh.yml => vtpc.yml} (76%) diff --git a/.github/workflows/vtsh.yml b/.github/workflows/vtpc.yml similarity index 76% rename from .github/workflows/vtsh.yml rename to .github/workflows/vtpc.yml index 04ca522..183b34f 100644 --- a/.github/workflows/vtsh.yml +++ b/.github/workflows/vtpc.yml @@ -1,4 +1,4 @@ -name: Test VT Shell + name: Test VT Page Cache on: push: @@ -6,7 +6,7 @@ on: - main pull_request: paths: - - 'lab/vtsh/**' + - 'lab/vtpc/**' jobs: build: @@ -15,12 +15,12 @@ jobs: cmake_build_type: - Asan - Release - runs-on: self-hosted + runs-on: ubuntu-latest container: image: silkeh/clang:latest defaults: run: - working-directory: ./lab/vtsh + working-directory: ./lab/vtpc steps: - name: Checkout uses: actions/checkout@v4 @@ -44,6 +44,11 @@ jobs: \( -iname '*.c' -o -iname '*.h' -o -iname '*.cpp' -o -iname '*.hpp' \) -exec \ clang-tidy -p ./build {} \; - - name: Test - working-directory: ./lab/vtsh/test - run: python3 -m unittest discover \ No newline at end of file + - name: Test Basic + run: ./build/test/test_basic + + - name: Test Sequential + run: ./build/test/test_seq + + - name: Test Random + run: ./build/test/test_random \ No newline at end of file From f3232c311a684cdad6e1f2f1b1620e8eac6b5c8b Mon Sep 17 00:00:00 2001 From: Ivan Zinchenko Date: Fri, 13 Feb 2026 22:04:58 +0300 Subject: [PATCH 3/7] Fix workflow --- .github/workflows/vtpc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vtpc.yml b/.github/workflows/vtpc.yml index 183b34f..67ab7b7 100644 --- a/.github/workflows/vtpc.yml +++ b/.github/workflows/vtpc.yml @@ -1,4 +1,4 @@ - name: Test VT Page Cache +name: Test VT Page Cache on: push: From b0bd17ca5846b4cdf19b13cb3337f6fbfd0e1e77 Mon Sep 17 00:00:00 2001 From: Ivan Zinchenko Date: Fri, 13 Feb 2026 22:14:24 +0300 Subject: [PATCH 4/7] WIP --- lab/vtpc/lib/CMakeLists.txt | 3 + lab/vtpc/lib/vtpc.c | 494 +---------------------------------- lab/vtpc/lib/vtpc_cache.c | 313 ++++++++++++++++++++++ lab/vtpc/lib/vtpc_internal.h | 96 +++++++ lab/vtpc/lib/vtpc_list.c | 83 ++++++ lab/vtpc/lib/vtpc_state.c | 52 ++++ 6 files changed, 550 insertions(+), 491 deletions(-) create mode 100644 lab/vtpc/lib/vtpc_cache.c create mode 100644 lab/vtpc/lib/vtpc_internal.h create mode 100644 lab/vtpc/lib/vtpc_list.c create mode 100644 lab/vtpc/lib/vtpc_state.c diff --git a/lab/vtpc/lib/CMakeLists.txt b/lab/vtpc/lib/CMakeLists.txt index e904b37..d565ca5 100644 --- a/lab/vtpc/lib/CMakeLists.txt +++ b/lab/vtpc/lib/CMakeLists.txt @@ -2,6 +2,9 @@ add_library( vtpc STATIC vtpc.c + vtpc_state.c + vtpc_list.c + vtpc_cache.c ) target_include_directories( diff --git a/lab/vtpc/lib/vtpc.c b/lab/vtpc/lib/vtpc.c index 1940500..acb775c 100644 --- a/lab/vtpc/lib/vtpc.c +++ b/lab/vtpc/lib/vtpc.c @@ -2,501 +2,12 @@ #include #include -#include -#include -#include -#include #include #include #include #include -#define VTPC_MAX_OPEN_FILES 256 -#define VTPC_MAX_CACHE_PAGES 128 -#define VTPC_MAX_A1OUT_PAGES 128 - -enum vtpc_resident_list { - VTPC_LIST_NONE = 0, - VTPC_LIST_A1IN = 1, - VTPC_LIST_AM = 2, -}; - -typedef struct { - int head; - int tail; - size_t size; -} vtpc_list; - -typedef struct { - int used; - int prev; - int next; - int list; - int dirty; - int file_slot; - off_t page_no; - unsigned char* data; -} vtpc_resident_page; - -typedef struct { - int used; - int prev; - int next; - int file_slot; - off_t page_no; -} vtpc_ghost_page; - -typedef struct { - int used; - int os_fd; - int open_flags; - off_t pos; - off_t size; - int size_dirty; -} vtpc_file; - -typedef struct { - int initialized; - size_t page_size; - size_t max_resident; - size_t max_a1out; - size_t kin; - - vtpc_list a1in; - vtpc_list am; - vtpc_list a1out; - - vtpc_file files[VTPC_MAX_OPEN_FILES]; - vtpc_resident_page resident[VTPC_MAX_CACHE_PAGES]; - vtpc_ghost_page ghosts[VTPC_MAX_A1OUT_PAGES]; -} vtpc_state; - -static vtpc_state g_state = {0}; - -static size_t vtpc_min_size(size_t lhs, size_t rhs) { - return lhs < rhs ? lhs : rhs; -} - -static int vtpc_open_fallback(const char* path, int mode, int access) { - int fd = open(path, mode, access); - return fd; -} - -static int vtpc_state_init(void) { - if (g_state.initialized) { - return 0; - } - - long page_sz = sysconf(_SC_PAGESIZE); - if (page_sz <= 0) { - page_sz = 4096; - } - - g_state.page_size = (size_t)page_sz; - g_state.max_resident = VTPC_MAX_CACHE_PAGES; - g_state.max_a1out = VTPC_MAX_A1OUT_PAGES; - g_state.kin = g_state.max_resident / 4; - if (g_state.kin == 0) { - g_state.kin = 1; - } - - g_state.a1in.head = -1; - g_state.a1in.tail = -1; - g_state.am.head = -1; - g_state.am.tail = -1; - g_state.a1out.head = -1; - g_state.a1out.tail = -1; - - g_state.initialized = 1; - return 0; -} - -static void vtpc_resident_list_push_back(vtpc_list* list, int idx) { - vtpc_resident_page* page = &g_state.resident[idx]; - page->prev = list->tail; - page->next = -1; - if (list->tail != -1) { - g_state.resident[list->tail].next = idx; - } else { - list->head = idx; - } - list->tail = idx; - ++list->size; -} - -static void vtpc_resident_list_remove(vtpc_list* list, int idx) { - vtpc_resident_page* page = &g_state.resident[idx]; - if (page->prev != -1) { - g_state.resident[page->prev].next = page->next; - } else { - list->head = page->next; - } - if (page->next != -1) { - g_state.resident[page->next].prev = page->prev; - } else { - list->tail = page->prev; - } - page->prev = -1; - page->next = -1; - if (list->size > 0) { - --list->size; - } -} - -static int vtpc_resident_list_pop_front(vtpc_list* list) { - if (list->head == -1) { - return -1; - } - int idx = list->head; - vtpc_resident_list_remove(list, idx); - return idx; -} - -static void vtpc_ghost_list_push_back(vtpc_list* list, int idx) { - vtpc_ghost_page* page = &g_state.ghosts[idx]; - page->prev = list->tail; - page->next = -1; - if (list->tail != -1) { - g_state.ghosts[list->tail].next = idx; - } else { - list->head = idx; - } - list->tail = idx; - ++list->size; -} - -static void vtpc_ghost_list_remove(vtpc_list* list, int idx) { - vtpc_ghost_page* page = &g_state.ghosts[idx]; - if (page->prev != -1) { - g_state.ghosts[page->prev].next = page->next; - } else { - list->head = page->next; - } - if (page->next != -1) { - g_state.ghosts[page->next].prev = page->prev; - } else { - list->tail = page->prev; - } - page->prev = -1; - page->next = -1; - if (list->size > 0) { - --list->size; - } -} - -static int vtpc_ghost_list_pop_front(vtpc_list* list) { - if (list->head == -1) { - return -1; - } - int idx = list->head; - vtpc_ghost_list_remove(list, idx); - return idx; -} - -static vtpc_file* vtpc_get_file(int fd) { - if (fd < 0 || fd >= VTPC_MAX_OPEN_FILES || !g_state.files[fd].used) { - errno = EBADF; - return NULL; - } - return &g_state.files[fd]; -} - -static int vtpc_find_resident_page(int file_slot, off_t page_no) { - for (int i = 0; i < VTPC_MAX_CACHE_PAGES; ++i) { - vtpc_resident_page* page = &g_state.resident[i]; - if (page->used && page->file_slot == file_slot && page->page_no == page_no) { - return i; - } - } - return -1; -} - -static int vtpc_find_ghost_page(int file_slot, off_t page_no) { - for (int i = 0; i < VTPC_MAX_A1OUT_PAGES; ++i) { - vtpc_ghost_page* page = &g_state.ghosts[i]; - if (page->used && page->file_slot == file_slot && page->page_no == page_no) { - return i; - } - } - return -1; -} - -static int vtpc_alloc_resident_slot(void) { - for (int i = 0; i < VTPC_MAX_CACHE_PAGES; ++i) { - vtpc_resident_page* page = &g_state.resident[i]; - if (!page->used) { - if (page->data == NULL) { - if (posix_memalign((void**)&page->data, g_state.page_size, g_state.page_size) != 0) { - errno = ENOMEM; - return -1; - } - } - page->used = 1; - page->prev = -1; - page->next = -1; - page->list = VTPC_LIST_NONE; - page->dirty = 0; - return i; - } - } - errno = ENOMEM; - return -1; -} - -static int vtpc_alloc_ghost_slot(void) { - for (int i = 0; i < VTPC_MAX_A1OUT_PAGES; ++i) { - if (!g_state.ghosts[i].used) { - g_state.ghosts[i].used = 1; - g_state.ghosts[i].prev = -1; - g_state.ghosts[i].next = -1; - return i; - } - } - errno = ENOMEM; - return -1; -} - -static int vtpc_pread_page(int os_fd, off_t page_no, unsigned char* data) { - const off_t offset = (off_t)(page_no * (off_t)g_state.page_size); - size_t done = 0; - while (done < g_state.page_size) { - ssize_t rc = pread(os_fd, data + done, g_state.page_size - done, offset + (off_t)done); - if (rc < 0) { - if (errno == EINTR) { - continue; - } - return -1; - } - if (rc == 0) { - memset(data + done, 0, g_state.page_size - done); - break; - } - done += (size_t)rc; - } - return 0; -} - -static int vtpc_pwrite_page(int os_fd, off_t page_no, const unsigned char* data) { - const off_t offset = (off_t)(page_no * (off_t)g_state.page_size); - size_t done = 0; - while (done < g_state.page_size) { - ssize_t rc = pwrite(os_fd, data + done, g_state.page_size - done, offset + (off_t)done); - if (rc < 0) { - if (errno == EINTR) { - continue; - } - return -1; - } - if (rc == 0) { - errno = EIO; - return -1; - } - done += (size_t)rc; - } - return 0; -} - -static int vtpc_flush_page(int idx) { - vtpc_resident_page* page = &g_state.resident[idx]; - if (!page->used || !page->dirty) { - return 0; - } - - vtpc_file* file = &g_state.files[page->file_slot]; - if (!file->used) { - errno = EBADF; - return -1; - } - - if (vtpc_pwrite_page(file->os_fd, page->page_no, page->data) < 0) { - return -1; - } - page->dirty = 0; - return 0; -} - -static void vtpc_trim_a1out(void) { - while (g_state.a1out.size > g_state.max_a1out) { - int idx = vtpc_ghost_list_pop_front(&g_state.a1out); - if (idx == -1) { - break; - } - g_state.ghosts[idx].used = 0; - } -} - -static int vtpc_evict_one(void) { - int victim = -1; - int victim_from_a1in = 0; - - if (g_state.a1in.size > g_state.kin) { - victim = vtpc_resident_list_pop_front(&g_state.a1in); - victim_from_a1in = 1; - } else if (g_state.am.size > 0) { - victim = vtpc_resident_list_pop_front(&g_state.am); - } else { - victim = vtpc_resident_list_pop_front(&g_state.a1in); - victim_from_a1in = 1; - } - - if (victim == -1) { - errno = ENOMEM; - return -1; - } - - vtpc_resident_page* page = &g_state.resident[victim]; - page->list = VTPC_LIST_NONE; - - if (vtpc_flush_page(victim) < 0) { - return -1; - } - - if (victim_from_a1in) { - int ghost_idx = vtpc_alloc_ghost_slot(); - if (ghost_idx >= 0) { - g_state.ghosts[ghost_idx].file_slot = page->file_slot; - g_state.ghosts[ghost_idx].page_no = page->page_no; - vtpc_ghost_list_push_back(&g_state.a1out, ghost_idx); - vtpc_trim_a1out(); - } - } - - page->used = 0; - page->dirty = 0; - page->file_slot = -1; - page->page_no = 0; - return 0; -} - -static int vtpc_ensure_resident_capacity(void) { - while (g_state.a1in.size + g_state.am.size >= g_state.max_resident) { - if (vtpc_evict_one() < 0) { - return -1; - } - } - return 0; -} - -static void vtpc_move_resident_to_am(int idx) { - vtpc_resident_page* page = &g_state.resident[idx]; - if (page->list == VTPC_LIST_A1IN) { - vtpc_resident_list_remove(&g_state.a1in, idx); - } else if (page->list == VTPC_LIST_AM) { - vtpc_resident_list_remove(&g_state.am, idx); - } - page->list = VTPC_LIST_AM; - vtpc_resident_list_push_back(&g_state.am, idx); -} - -static int vtpc_get_or_load_page(int file_slot, off_t page_no, int need_existing, int* out_idx) { - int resident_idx = vtpc_find_resident_page(file_slot, page_no); - if (resident_idx >= 0) { - if (g_state.resident[resident_idx].list == VTPC_LIST_A1IN) { - vtpc_move_resident_to_am(resident_idx); - } else if (g_state.resident[resident_idx].list == VTPC_LIST_AM) { - vtpc_resident_list_remove(&g_state.am, resident_idx); - vtpc_resident_list_push_back(&g_state.am, resident_idx); - } - *out_idx = resident_idx; - return 0; - } - - if (vtpc_ensure_resident_capacity() < 0) { - return -1; - } - - int ghost_idx = vtpc_find_ghost_page(file_slot, page_no); - int idx = vtpc_alloc_resident_slot(); - if (idx < 0) { - return -1; - } - - vtpc_resident_page* page = &g_state.resident[idx]; - page->file_slot = file_slot; - page->page_no = page_no; - page->dirty = 0; - - if (need_existing) { - vtpc_file* file = &g_state.files[file_slot]; - if (vtpc_pread_page(file->os_fd, page_no, page->data) < 0) { - page->used = 0; - return -1; - } - } else { - memset(page->data, 0, g_state.page_size); - } - - if (ghost_idx >= 0) { - vtpc_ghost_list_remove(&g_state.a1out, ghost_idx); - g_state.ghosts[ghost_idx].used = 0; - page->list = VTPC_LIST_AM; - vtpc_resident_list_push_back(&g_state.am, idx); - } else { - page->list = VTPC_LIST_A1IN; - vtpc_resident_list_push_back(&g_state.a1in, idx); - } - - *out_idx = idx; - return 0; -} - -static int vtpc_flush_file_pages(vtpc_file* file, int file_slot, int with_fsync) { - for (int i = 0; i < VTPC_MAX_CACHE_PAGES; ++i) { - vtpc_resident_page* page = &g_state.resident[i]; - if (page->used && page->file_slot == file_slot && page->dirty) { - if (vtpc_flush_page(i) < 0) { - return -1; - } - } - } - - if (file->size_dirty) { - if (ftruncate(file->os_fd, file->size) < 0) { - return -1; - } - } - - if (with_fsync) { - if (fsync(file->os_fd) < 0) { - return -1; - } - } - - file->size_dirty = 0; - return 0; -} - -static void vtpc_drop_file_pages(int file_slot) { - for (int i = 0; i < VTPC_MAX_CACHE_PAGES; ++i) { - vtpc_resident_page* page = &g_state.resident[i]; - if (!page->used || page->file_slot != file_slot) { - continue; - } - - if (page->list == VTPC_LIST_A1IN) { - vtpc_resident_list_remove(&g_state.a1in, i); - } else if (page->list == VTPC_LIST_AM) { - vtpc_resident_list_remove(&g_state.am, i); - } - - page->used = 0; - page->dirty = 0; - page->list = VTPC_LIST_NONE; - page->file_slot = -1; - page->page_no = 0; - } - - for (int i = 0; i < VTPC_MAX_A1OUT_PAGES; ++i) { - vtpc_ghost_page* ghost = &g_state.ghosts[i]; - if (!ghost->used || ghost->file_slot != file_slot) { - continue; - } - vtpc_ghost_list_remove(&g_state.a1out, i); - ghost->used = 0; - ghost->file_slot = -1; - ghost->page_no = 0; - } -} +#include "vtpc_internal.h" int vtpc_open(const char* path, int mode, int access) { if (vtpc_state_init() < 0) { @@ -510,7 +21,8 @@ int vtpc_open(const char* path, int mode, int access) { int os_fd = open(path, open_mode, access); #ifdef O_DIRECT - if (os_fd < 0 && (errno == EINVAL || errno == EOPNOTSUPP || errno == ENOTSUP)) { + if (os_fd < 0 && + (errno == EINVAL || errno == EOPNOTSUPP || errno == ENOTSUP)) { os_fd = vtpc_open_fallback(path, mode, access); } #endif diff --git a/lab/vtpc/lib/vtpc_cache.c b/lab/vtpc/lib/vtpc_cache.c new file mode 100644 index 0000000..1eb8e73 --- /dev/null +++ b/lab/vtpc/lib/vtpc_cache.c @@ -0,0 +1,313 @@ +#include +#include +#include +#include + +#include "vtpc_internal.h" + +int vtpc_find_resident_page(int file_slot, off_t page_no) { + for (int i = 0; i < VTPC_MAX_CACHE_PAGES; ++i) { + vtpc_resident_page* page = &g_state.resident[i]; + if (page->used && page->file_slot == file_slot && + page->page_no == page_no) { + return i; + } + } + return -1; +} + +int vtpc_find_ghost_page(int file_slot, off_t page_no) { + for (int i = 0; i < VTPC_MAX_A1OUT_PAGES; ++i) { + vtpc_ghost_page* page = &g_state.ghosts[i]; + if (page->used && page->file_slot == file_slot && + page->page_no == page_no) { + return i; + } + } + return -1; +} + +int vtpc_alloc_resident_slot(void) { + for (int i = 0; i < VTPC_MAX_CACHE_PAGES; ++i) { + vtpc_resident_page* page = &g_state.resident[i]; + if (!page->used) { + if (page->data == NULL) { + if (posix_memalign( + (void**)&page->data, g_state.page_size, g_state.page_size + ) != 0) { + errno = ENOMEM; + return -1; + } + } + page->used = 1; + page->prev = -1; + page->next = -1; + page->list = VTPC_LIST_NONE; + page->dirty = 0; + return i; + } + } + errno = ENOMEM; + return -1; +} + +int vtpc_alloc_ghost_slot(void) { + for (int i = 0; i < VTPC_MAX_A1OUT_PAGES; ++i) { + if (!g_state.ghosts[i].used) { + g_state.ghosts[i].used = 1; + g_state.ghosts[i].prev = -1; + g_state.ghosts[i].next = -1; + return i; + } + } + errno = ENOMEM; + return -1; +} + +int vtpc_pread_page(int os_fd, off_t page_no, unsigned char* data) { + const off_t offset = (off_t)(page_no * (off_t)g_state.page_size); + size_t done = 0; + while (done < g_state.page_size) { + ssize_t rc = pread( + os_fd, data + done, g_state.page_size - done, offset + (off_t)done + ); + if (rc < 0) { + if (errno == EINTR) { + continue; + } + return -1; + } + if (rc == 0) { + memset(data + done, 0, g_state.page_size - done); + break; + } + done += (size_t)rc; + } + return 0; +} + +int vtpc_pwrite_page(int os_fd, off_t page_no, const unsigned char* data) { + const off_t offset = (off_t)(page_no * (off_t)g_state.page_size); + size_t done = 0; + while (done < g_state.page_size) { + ssize_t rc = pwrite( + os_fd, data + done, g_state.page_size - done, offset + (off_t)done + ); + if (rc < 0) { + if (errno == EINTR) { + continue; + } + return -1; + } + if (rc == 0) { + errno = EIO; + return -1; + } + done += (size_t)rc; + } + return 0; +} + +int vtpc_flush_page(int idx) { + vtpc_resident_page* page = &g_state.resident[idx]; + if (!page->used || !page->dirty) { + return 0; + } + + vtpc_file* file = &g_state.files[page->file_slot]; + if (!file->used) { + errno = EBADF; + return -1; + } + + if (vtpc_pwrite_page(file->os_fd, page->page_no, page->data) < 0) { + return -1; + } + page->dirty = 0; + return 0; +} + +void vtpc_trim_a1out(void) { + while (g_state.a1out.size > g_state.max_a1out) { + int idx = vtpc_ghost_list_pop_front(&g_state.a1out); + if (idx == -1) { + break; + } + g_state.ghosts[idx].used = 0; + } +} + +int vtpc_evict_one(void) { + int victim = -1; + int victim_from_a1in = 0; + + if (g_state.a1in.size > g_state.kin) { + victim = vtpc_resident_list_pop_front(&g_state.a1in); + victim_from_a1in = 1; + } else if (g_state.am.size > 0) { + victim = vtpc_resident_list_pop_front(&g_state.am); + } else { + victim = vtpc_resident_list_pop_front(&g_state.a1in); + victim_from_a1in = 1; + } + + if (victim == -1) { + errno = ENOMEM; + return -1; + } + + vtpc_resident_page* page = &g_state.resident[victim]; + page->list = VTPC_LIST_NONE; + + if (vtpc_flush_page(victim) < 0) { + return -1; + } + + if (victim_from_a1in) { + int ghost_idx = vtpc_alloc_ghost_slot(); + if (ghost_idx >= 0) { + g_state.ghosts[ghost_idx].file_slot = page->file_slot; + g_state.ghosts[ghost_idx].page_no = page->page_no; + vtpc_ghost_list_push_back(&g_state.a1out, ghost_idx); + vtpc_trim_a1out(); + } + } + + page->used = 0; + page->dirty = 0; + page->file_slot = -1; + page->page_no = 0; + return 0; +} + +int vtpc_ensure_resident_capacity(void) { + while (g_state.a1in.size + g_state.am.size >= g_state.max_resident) { + if (vtpc_evict_one() < 0) { + return -1; + } + } + return 0; +} + +void vtpc_move_resident_to_am(int idx) { + vtpc_resident_page* page = &g_state.resident[idx]; + if (page->list == VTPC_LIST_A1IN) { + vtpc_resident_list_remove(&g_state.a1in, idx); + } else if (page->list == VTPC_LIST_AM) { + vtpc_resident_list_remove(&g_state.am, idx); + } + page->list = VTPC_LIST_AM; + vtpc_resident_list_push_back(&g_state.am, idx); +} + +int vtpc_get_or_load_page( + int file_slot, off_t page_no, int need_existing, int* out_idx +) { + int resident_idx = vtpc_find_resident_page(file_slot, page_no); + if (resident_idx >= 0) { + if (g_state.resident[resident_idx].list == VTPC_LIST_A1IN) { + vtpc_move_resident_to_am(resident_idx); + } else if (g_state.resident[resident_idx].list == VTPC_LIST_AM) { + vtpc_resident_list_remove(&g_state.am, resident_idx); + vtpc_resident_list_push_back(&g_state.am, resident_idx); + } + *out_idx = resident_idx; + return 0; + } + + if (vtpc_ensure_resident_capacity() < 0) { + return -1; + } + + int ghost_idx = vtpc_find_ghost_page(file_slot, page_no); + int idx = vtpc_alloc_resident_slot(); + if (idx < 0) { + return -1; + } + + vtpc_resident_page* page = &g_state.resident[idx]; + page->file_slot = file_slot; + page->page_no = page_no; + page->dirty = 0; + + if (need_existing) { + vtpc_file* file = &g_state.files[file_slot]; + if (vtpc_pread_page(file->os_fd, page_no, page->data) < 0) { + page->used = 0; + return -1; + } + } else { + memset(page->data, 0, g_state.page_size); + } + + if (ghost_idx >= 0) { + vtpc_ghost_list_remove(&g_state.a1out, ghost_idx); + g_state.ghosts[ghost_idx].used = 0; + page->list = VTPC_LIST_AM; + vtpc_resident_list_push_back(&g_state.am, idx); + } else { + page->list = VTPC_LIST_A1IN; + vtpc_resident_list_push_back(&g_state.a1in, idx); + } + + *out_idx = idx; + return 0; +} + +int vtpc_flush_file_pages(vtpc_file* file, int file_slot, int with_fsync) { + for (int i = 0; i < VTPC_MAX_CACHE_PAGES; ++i) { + vtpc_resident_page* page = &g_state.resident[i]; + if (page->used && page->file_slot == file_slot && page->dirty) { + if (vtpc_flush_page(i) < 0) { + return -1; + } + } + } + + if (file->size_dirty) { + if (ftruncate(file->os_fd, file->size) < 0) { + return -1; + } + } + + if (with_fsync) { + if (fsync(file->os_fd) < 0) { + return -1; + } + } + + file->size_dirty = 0; + return 0; +} + +void vtpc_drop_file_pages(int file_slot) { + for (int i = 0; i < VTPC_MAX_CACHE_PAGES; ++i) { + vtpc_resident_page* page = &g_state.resident[i]; + if (!page->used || page->file_slot != file_slot) { + continue; + } + + if (page->list == VTPC_LIST_A1IN) { + vtpc_resident_list_remove(&g_state.a1in, i); + } else if (page->list == VTPC_LIST_AM) { + vtpc_resident_list_remove(&g_state.am, i); + } + + page->used = 0; + page->dirty = 0; + page->list = VTPC_LIST_NONE; + page->file_slot = -1; + page->page_no = 0; + } + + for (int i = 0; i < VTPC_MAX_A1OUT_PAGES; ++i) { + vtpc_ghost_page* ghost = &g_state.ghosts[i]; + if (!ghost->used || ghost->file_slot != file_slot) { + continue; + } + vtpc_ghost_list_remove(&g_state.a1out, i); + ghost->used = 0; + ghost->file_slot = -1; + ghost->page_no = 0; + } +} diff --git a/lab/vtpc/lib/vtpc_internal.h b/lab/vtpc/lib/vtpc_internal.h new file mode 100644 index 0000000..1f00cdd --- /dev/null +++ b/lab/vtpc/lib/vtpc_internal.h @@ -0,0 +1,96 @@ +#pragma once + +#include +#include + +#define VTPC_MAX_OPEN_FILES 256 +#define VTPC_MAX_CACHE_PAGES 128 +#define VTPC_MAX_A1OUT_PAGES 128 + +enum vtpc_resident_list { + VTPC_LIST_NONE = 0, + VTPC_LIST_A1IN = 1, + VTPC_LIST_AM = 2, +}; + +typedef struct { + int head; + int tail; + size_t size; +} vtpc_list; + +typedef struct { + int used; + int prev; + int next; + int list; + int dirty; + int file_slot; + off_t page_no; + unsigned char* data; +} vtpc_resident_page; + +typedef struct { + int used; + int prev; + int next; + int file_slot; + off_t page_no; +} vtpc_ghost_page; + +typedef struct { + int used; + int os_fd; + int open_flags; + off_t pos; + off_t size; + int size_dirty; +} vtpc_file; + +typedef struct { + int initialized; + size_t page_size; + size_t max_resident; + size_t max_a1out; + size_t kin; + + vtpc_list a1in; + vtpc_list am; + vtpc_list a1out; + + vtpc_file files[VTPC_MAX_OPEN_FILES]; + vtpc_resident_page resident[VTPC_MAX_CACHE_PAGES]; + vtpc_ghost_page ghosts[VTPC_MAX_A1OUT_PAGES]; +} vtpc_state; + +extern vtpc_state g_state; + +size_t vtpc_min_size(size_t lhs, size_t rhs); +int vtpc_open_fallback(const char* path, int mode, int access); +int vtpc_state_init(void); +vtpc_file* vtpc_get_file(int fd); + +void vtpc_resident_list_push_back(vtpc_list* list, int idx); +void vtpc_resident_list_remove(vtpc_list* list, int idx); +int vtpc_resident_list_pop_front(vtpc_list* list); + +void vtpc_ghost_list_push_back(vtpc_list* list, int idx); +void vtpc_ghost_list_remove(vtpc_list* list, int idx); +int vtpc_ghost_list_pop_front(vtpc_list* list); + +int vtpc_find_resident_page(int file_slot, off_t page_no); +int vtpc_find_ghost_page(int file_slot, off_t page_no); +int vtpc_alloc_resident_slot(void); +int vtpc_alloc_ghost_slot(void); +int vtpc_pread_page(int os_fd, off_t page_no, unsigned char* data); +int vtpc_pwrite_page(int os_fd, off_t page_no, const unsigned char* data); +int vtpc_flush_page(int idx); +void vtpc_trim_a1out(void); +int vtpc_evict_one(void); +int vtpc_ensure_resident_capacity(void); +void vtpc_move_resident_to_am(int idx); +int vtpc_get_or_load_page( + int file_slot, off_t page_no, int need_existing, int* out_idx +); +int vtpc_flush_file_pages(vtpc_file* file, int file_slot, int with_fsync); +void vtpc_drop_file_pages(int file_slot); diff --git a/lab/vtpc/lib/vtpc_list.c b/lab/vtpc/lib/vtpc_list.c new file mode 100644 index 0000000..5c37e9f --- /dev/null +++ b/lab/vtpc/lib/vtpc_list.c @@ -0,0 +1,83 @@ +#include "vtpc_internal.h" + +void vtpc_resident_list_push_back(vtpc_list* list, int idx) { + vtpc_resident_page* page = &g_state.resident[idx]; + page->prev = list->tail; + page->next = -1; + if (list->tail != -1) { + g_state.resident[list->tail].next = idx; + } else { + list->head = idx; + } + list->tail = idx; + ++list->size; +} + +void vtpc_resident_list_remove(vtpc_list* list, int idx) { + vtpc_resident_page* page = &g_state.resident[idx]; + if (page->prev != -1) { + g_state.resident[page->prev].next = page->next; + } else { + list->head = page->next; + } + if (page->next != -1) { + g_state.resident[page->next].prev = page->prev; + } else { + list->tail = page->prev; + } + page->prev = -1; + page->next = -1; + if (list->size > 0) { + --list->size; + } +} + +int vtpc_resident_list_pop_front(vtpc_list* list) { + if (list->head == -1) { + return -1; + } + int idx = list->head; + vtpc_resident_list_remove(list, idx); + return idx; +} + +void vtpc_ghost_list_push_back(vtpc_list* list, int idx) { + vtpc_ghost_page* page = &g_state.ghosts[idx]; + page->prev = list->tail; + page->next = -1; + if (list->tail != -1) { + g_state.ghosts[list->tail].next = idx; + } else { + list->head = idx; + } + list->tail = idx; + ++list->size; +} + +void vtpc_ghost_list_remove(vtpc_list* list, int idx) { + vtpc_ghost_page* page = &g_state.ghosts[idx]; + if (page->prev != -1) { + g_state.ghosts[page->prev].next = page->next; + } else { + list->head = page->next; + } + if (page->next != -1) { + g_state.ghosts[page->next].prev = page->prev; + } else { + list->tail = page->prev; + } + page->prev = -1; + page->next = -1; + if (list->size > 0) { + --list->size; + } +} + +int vtpc_ghost_list_pop_front(vtpc_list* list) { + if (list->head == -1) { + return -1; + } + int idx = list->head; + vtpc_ghost_list_remove(list, idx); + return idx; +} diff --git a/lab/vtpc/lib/vtpc_state.c b/lab/vtpc/lib/vtpc_state.c new file mode 100644 index 0000000..4a0bcce --- /dev/null +++ b/lab/vtpc/lib/vtpc_state.c @@ -0,0 +1,52 @@ +#include +#include +#include + +#include "vtpc_internal.h" + +vtpc_state g_state = {0}; + +size_t vtpc_min_size(size_t lhs, size_t rhs) { + return lhs < rhs ? lhs : rhs; +} + +int vtpc_open_fallback(const char* path, int mode, int access) { + return open(path, mode, access); +} + +int vtpc_state_init(void) { + if (g_state.initialized) { + return 0; + } + + long page_sz = sysconf(_SC_PAGESIZE); + if (page_sz <= 0) { + page_sz = 4096; + } + + g_state.page_size = (size_t)page_sz; + g_state.max_resident = VTPC_MAX_CACHE_PAGES; + g_state.max_a1out = VTPC_MAX_A1OUT_PAGES; + g_state.kin = g_state.max_resident / 4; + if (g_state.kin == 0) { + g_state.kin = 1; + } + + g_state.a1in.head = -1; + g_state.a1in.tail = -1; + g_state.am.head = -1; + g_state.am.tail = -1; + g_state.a1out.head = -1; + g_state.a1out.tail = -1; + + g_state.initialized = 1; + return 0; +} + +vtpc_file* vtpc_get_file(int fd) { + if (fd < 0 || fd >= VTPC_MAX_OPEN_FILES || !g_state.files[fd].used) { + errno = EBADF; + return NULL; + } + return &g_state.files[fd]; +} From b292b7f2ae18919abab7718ac79dc244b07ae46c Mon Sep 17 00:00:00 2001 From: Ivan Zinchenko Date: Sat, 14 Feb 2026 01:06:09 +0300 Subject: [PATCH 5/7] WIP --- lab/vtpc/lib/vtpc.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lab/vtpc/lib/vtpc.c b/lab/vtpc/lib/vtpc.c index acb775c..e91ce30 100644 --- a/lab/vtpc/lib/vtpc.c +++ b/lab/vtpc/lib/vtpc.c @@ -15,17 +15,13 @@ int vtpc_open(const char* path, int mode, int access) { } int open_mode = mode; -#ifdef O_DIRECT open_mode |= O_DIRECT; -#endif int os_fd = open(path, open_mode, access); -#ifdef O_DIRECT if (os_fd < 0 && (errno == EINVAL || errno == EOPNOTSUPP || errno == ENOTSUP)) { os_fd = vtpc_open_fallback(path, mode, access); } -#endif if (os_fd < 0) { return -1; } From acad738f12b5070a22ddb3483b5c3dd629bc3130 Mon Sep 17 00:00:00 2001 From: Ivan Zinchenko Date: Sat, 14 Feb 2026 01:58:45 +0300 Subject: [PATCH 6/7] WIP --- lab/vtpc/CMakeLists.txt | 1 + lab/vtpc/lib/CMakeLists.txt | 6 + lab/vtpc/lib/vtpc.c | 10 +- lab/vtpc/lib/vtpc_cache.c | 2 + lab/vtpc/lib/vtpc_internal.h | 8 +- lab/vtpc/lib/vtpc_state.c | 5 - lab/vtpc/vtld/.clang-format | 16 +++ lab/vtpc/vtld/.gitignore | 5 + lab/vtpc/vtld/.vscode/settings.json | 5 + lab/vtpc/vtld/README.md | 165 +++++++++++++++++++++++ lab/vtpc/vtld/ema_replace_str.c | 201 ++++++++++++++++++++++++++++ lab/vtpc/vtld/ema_utils.c | 126 +++++++++++++++++ lab/vtpc/vtld/ema_utils.h | 14 ++ lab/vtpc/vtld/makefile | 10 ++ lab/vtpc/vtld/util.c | 21 +++ lab/vtpc/vtld/util.h | 6 + 16 files changed, 588 insertions(+), 13 deletions(-) create mode 100644 lab/vtpc/vtld/.clang-format create mode 100644 lab/vtpc/vtld/.gitignore create mode 100644 lab/vtpc/vtld/.vscode/settings.json create mode 100644 lab/vtpc/vtld/README.md create mode 100644 lab/vtpc/vtld/ema_replace_str.c create mode 100644 lab/vtpc/vtld/ema_utils.c create mode 100644 lab/vtpc/vtld/ema_utils.h create mode 100644 lab/vtpc/vtld/makefile create mode 100644 lab/vtpc/vtld/util.c create mode 100644 lab/vtpc/vtld/util.h diff --git a/lab/vtpc/CMakeLists.txt b/lab/vtpc/CMakeLists.txt index 478cbae..0dca19c 100644 --- a/lab/vtpc/CMakeLists.txt +++ b/lab/vtpc/CMakeLists.txt @@ -7,6 +7,7 @@ project( ) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +add_compile_definitions(_GNU_SOURCE) add_subdirectory(lib) add_subdirectory(test) diff --git a/lab/vtpc/lib/CMakeLists.txt b/lab/vtpc/lib/CMakeLists.txt index d565ca5..f5d3ab3 100644 --- a/lab/vtpc/lib/CMakeLists.txt +++ b/lab/vtpc/lib/CMakeLists.txt @@ -12,3 +12,9 @@ target_include_directories( PUBLIC . ) + +target_compile_definitions( + vtpc + PRIVATE + _GNU_SOURCE +) diff --git a/lab/vtpc/lib/vtpc.c b/lab/vtpc/lib/vtpc.c index e91ce30..9532a8b 100644 --- a/lab/vtpc/lib/vtpc.c +++ b/lab/vtpc/lib/vtpc.c @@ -14,14 +14,8 @@ int vtpc_open(const char* path, int mode, int access) { return -1; } - int open_mode = mode; - open_mode |= O_DIRECT; - + int open_mode = mode | O_DIRECT; int os_fd = open(path, open_mode, access); - if (os_fd < 0 && - (errno == EINVAL || errno == EOPNOTSUPP || errno == ENOTSUP)) { - os_fd = vtpc_open_fallback(path, mode, access); - } if (os_fd < 0) { return -1; } @@ -99,6 +93,7 @@ ssize_t vtpc_read(int fd, void* buf, size_t count) { return -1; } + // O_ACCMODE extracts access bits from open flags. int acc_mode = file->open_flags & O_ACCMODE; if (!(acc_mode == O_RDONLY || acc_mode == O_RDWR)) { errno = EBADF; @@ -204,6 +199,7 @@ off_t vtpc_lseek(int fd, off_t offset, int whence) { return -1; } + // whence defines the seek base: start, current position, or end. off_t base = 0; if (whence == SEEK_SET) { base = 0; diff --git a/lab/vtpc/lib/vtpc_cache.c b/lab/vtpc/lib/vtpc_cache.c index 1eb8e73..222c415 100644 --- a/lab/vtpc/lib/vtpc_cache.c +++ b/lab/vtpc/lib/vtpc_cache.c @@ -32,6 +32,7 @@ int vtpc_alloc_resident_slot(void) { vtpc_resident_page* page = &g_state.resident[i]; if (!page->used) { if (page->data == NULL) { + // O_DIRECT requires aligned buffers. if (posix_memalign( (void**)&page->data, g_state.page_size, g_state.page_size ) != 0) { @@ -137,6 +138,7 @@ void vtpc_trim_a1out(void) { } } +// Evict one page by 2Q policy and preserve metadata in A1out when needed. int vtpc_evict_one(void) { int victim = -1; int victim_from_a1in = 0; diff --git a/lab/vtpc/lib/vtpc_internal.h b/lab/vtpc/lib/vtpc_internal.h index 1f00cdd..64e98a0 100644 --- a/lab/vtpc/lib/vtpc_internal.h +++ b/lab/vtpc/lib/vtpc_internal.h @@ -20,12 +20,16 @@ typedef struct { } vtpc_list; typedef struct { + // Slot is occupied by a resident cached page. int used; int prev; int next; int list; + // Page was modified in cache and must be flushed. int dirty; + // Owner file index in g_state.files[]. int file_slot; + // File page number (offset / page_size). off_t page_no; unsigned char* data; } vtpc_resident_page; @@ -43,7 +47,9 @@ typedef struct { int os_fd; int open_flags; off_t pos; + // Logical file size tracked by vtpc. off_t size; + // 1 means ftruncate(size) is still pending. int size_dirty; } vtpc_file; @@ -52,6 +58,7 @@ typedef struct { size_t page_size; size_t max_resident; size_t max_a1out; + // 2Q threshold: preferred upper bound for A1in size. size_t kin; vtpc_list a1in; @@ -66,7 +73,6 @@ typedef struct { extern vtpc_state g_state; size_t vtpc_min_size(size_t lhs, size_t rhs); -int vtpc_open_fallback(const char* path, int mode, int access); int vtpc_state_init(void); vtpc_file* vtpc_get_file(int fd); diff --git a/lab/vtpc/lib/vtpc_state.c b/lab/vtpc/lib/vtpc_state.c index 4a0bcce..29a5663 100644 --- a/lab/vtpc/lib/vtpc_state.c +++ b/lab/vtpc/lib/vtpc_state.c @@ -1,5 +1,4 @@ #include -#include #include #include "vtpc_internal.h" @@ -10,10 +9,6 @@ size_t vtpc_min_size(size_t lhs, size_t rhs) { return lhs < rhs ? lhs : rhs; } -int vtpc_open_fallback(const char* path, int mode, int access) { - return open(path, mode, access); -} - int vtpc_state_init(void) { if (g_state.initialized) { return 0; diff --git a/lab/vtpc/vtld/.clang-format b/lab/vtpc/vtld/.clang-format new file mode 100644 index 0000000..dd99110 --- /dev/null +++ b/lab/vtpc/vtld/.clang-format @@ -0,0 +1,16 @@ +BasedOnStyle: Google + +AccessModifierOffset: -2 + +AllowShortFunctionsOnASingleLine: None +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false + +DerivePointerAlignment: false +BreakConstructorInitializers: BeforeComma +AlignAfterOpenBracket: BlockIndent + +BinPackArguments: false +BinPackParameters: false + +AlignArrayOfStructures: Right diff --git a/lab/vtpc/vtld/.gitignore b/lab/vtpc/vtld/.gitignore new file mode 100644 index 0000000..cfb1085 --- /dev/null +++ b/lab/vtpc/vtld/.gitignore @@ -0,0 +1,5 @@ +cpu_dedup +cpu_dedup_pthreads +ema_replace_str +ema_replace_str_pthreads +*.bin \ No newline at end of file diff --git a/lab/vtpc/vtld/.vscode/settings.json b/lab/vtpc/vtld/.vscode/settings.json new file mode 100644 index 0000000..f094915 --- /dev/null +++ b/lab/vtpc/vtld/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "files.associations": { + "errno.h": "c" + } +} \ No newline at end of file diff --git a/lab/vtpc/vtld/README.md b/lab/vtpc/vtld/README.md new file mode 100644 index 0000000..5f45463 --- /dev/null +++ b/lab/vtpc/vtld/README.md @@ -0,0 +1,165 @@ +# Базовый трек. Лабораторная работа 1 + +Вариант: `cpu-dedup ema-replace-str` + +## Часть 3. Анализ системы и мониторинг + +Разработайте параметризируемую программу-нагрузчик, которая будет однопоточно нагружать подсистему ввода-вывода (IO). Она должна принимать на вход следующие параметры: + +- `rw: read/write` - режим нагрузки: чтение или запись; +- `block_size: ` - размер блока в байтах, с которым производится чтение/запись; +- `block_count: ` - количество блоков; +- `file: ` - имя файла, с которым происходит работа; +- `range: -` - границы в пределах файла, в которые должны + осуществляться запись/чтение, значение по умолчанию, 0-0, означает, что доступен + весь файл; +- `direct: on/off` - открывать файл с опцией O_DIRECT (в обход кэшей ОС) или нет; +- `type: sequence/random` - режим выбора следующего блока для записи/чтения + последовательно или случайно; + +Разработайте комплекс программ-нагрузчиков согласно выданному варианту. +Каждый нагрузчик должен принимать параметр, который определяет +количество повторений для алгоритма, указанного в задании, а также другие +вспомогательные параметры. Варианты программ рассчитаны так, чтобы по-разному +нагружать вычислительную подсистему (CPU) и подсистему ввода-вывода (IO) - это +необходимо учитывать при их реализации. Разработанные программы необходимо +скомпилировать без дополнительных опций оптимизации компилятора. + +Проведите исследование поведения ОС во время исполнения разработанных +программ-нагрузчиков по следующему плану: + +1. Перед запуском нагрузчика, попробуйте оценить время работы вашей программы + или ее результаты (если по варианту вам досталось измерение чего либо) и + обоснуйте свои предположения. + +2. Запустите программу-нагрузчик и зафиксируйте метрики ее работы с помощью + инструментов для мониторинга и профилирования (см. лекции). Сравните полученные + результаты с ожидаемыми. Объяснените наблюдаемое поведение. Продолжительность + каждого запуска должна занимать достаточное для прекращения переходных + процессов время, по крайней мере, минуту. + +3. Определите количество одновременно запущенных процессов с программой-нагрузчиком, + которое эффективно нагружает все ядра процессора в вашей системе. Как + распределяются показатели времени USER%, SYS%, WAIT%, а также полное время + выполнения нагрузчика, какое количество переключений контекста (вынужденных и + невынужденных) происходит при этом? Подумайте над тем, как вы определяете + эффективность. + +4. Увеличьте количество нагрузчиков вдвое, втрое, вчетверо. Как изменились + исследуемые показатели? Почему? + +5. Объедините программы-нагрузчики в одну, реализованную при помощи потоков + выполнения, чтобы один нагрузчик эффективно нагружал все ядра вашей системы. Как + изменились показатели времени для того же объема вычислений? Запустите одну, + две, три таких программы. Как изменились исследуемые показатели? Почему? + +6. Скомпилируйте программу-нугрузчик с опцией агрессивной оптимизации. Как + изменились исследуемые показатели? На сколько сократилось реальное время + исполнения программы нагрузчика? Почему? + +В процессе защиты вашей работы преподаватель будет просить вас запустить +программу-нагрузчик с различными комбинациями параметров и просить объяснить +результат. + +## Требования к реализации + +1. Программа (комплекс программ) должна быть реализован на языке C. + +2. Дочерние процессы должны быть созданы через указанные в варианте системные + вызовы операционной системы, с обеспечением корректного запуска и завершения + процессов. Запрещено использовать высокоуровневые абстракции над системными + вызовами. Необходимо использовать, процедуры libc. + +## Требования к отчету и защите + +Отчет должен содержать: + +1. Титульный лист с указанием номера и названия ЛР, вашего ФИО, ФИО + преподавателя практики, номера вашей группы, варианта ЛР. + +2. Текст задания в соответствии с вариантом. + +3. Листинг исходного кода всех программ, написанных в рамках данной ЛР. + +4. Предположения о свойствах программ-нагрузчиков + +5. Результаты измерений и метрик программ-нагрузчиков, полученных инструментами + мониторинга. Должно быть описано, какие утилиты запускались, с какими + параметрами и их вывод. + +6. Сравнительный анализ ожидаемых и фактических значений исследованных показателей. + +7. Заключение. + +Студент должен быть готов: + +1. Продемонстрировать работоспособность Shell и предоставить исходный код. + +2. Воспроизвести ход работы по исследованию поведения ОС и продемонстрировать схожие + результаты работы программ-нагрузчиков. + +3. Ответить на вопросы по реализации программ и контрольные вопросы. + +## Описание вариантов + +Нагрузчик вычислительной подсистемы (группа **cpu**): + +- `cpu-factorize`: Разложение числа на простые множители. Задаваемое число + должно являться произведением нескольких больших простых чисел. + +- `cpu-sort`: Любая сортировка массива чисел за N*log(N) и любая за N**2. + Алгоритм должен быть реализован самим студентом, а не взят из стандартной + библиотеки. + +- `cpu-short-path`: Поиск кратчайшего пути в графе. Граф генерируется в момент + запуска или записан в исходный код. + +- `cpu-linreg`: Построение модели линейной регрессии для последовательности случайных чисел. Количество чисел и диапозон значений задаются параметром. + +- `cpu-dedup`: Дедупликация элементов в массиве. Массив генерируется на основе + случайных чисел. Размер массива задаётся параметром. + +- `cpu-mat-mul`: Перемножение матриц. Размер матриц задаётся параметром, а + элементы матриц генерируются на основе случайных чисел. + +- `cpu-calc-md5`: Подсчитать хеш md5 для текста полученного конкатенацией + фрагментов текста выбранных с помощью генератора случайных чисел. + +- `cpu-calc-crc`: Посчитать контрольную сумму crc для текста полученного + конкатенацией фрагментов текста выбранных с помощью генератора случайных чисел. + + +Нагрузчик, работающий с внешней памятью (группа **ema**): + +- `ema-replace-int`: Поиск элемента в массиве чисел во внешней памяти и замена + значения на другое. Файл для поиска значения генерируется в произвольном + формате, размер задаётся через параметр. + +- `ema-replace-str`: Поиск подстроки в тексте во внешней памяти и замена на + другую строку того же размера. Файл для поиска значения генерируется в + произвольном формате, размер задаётся через параметр. + +- `ema-sort-int`: Сортировка массива чисел во внешней памяти. Весь массив не + может быть загружен в память, поэтому сортировка происходит частями. + +- `ema-traverse-graph`: Обход k-регулярного направленного графа, + сериализованного в файл, и выполнение модифицирующей операции над определенной + вершиной графа. Поиск вершины происходит по хранимому в ней значению, возможно с + ограничением глубины поиска. + Формат хранения: вершина (struct Node) определяется её отступом от начала файла (id=index*(sizeof struct Node)), в вершине хранится одно значение типа фиксированного размера и k номеров вершин, в которые из неё можно попасть; в файле вершины хранятся последовательно с выравниванием соответствующим размеру структуры. Тип значения, хранимого в вершине, и значение k подбираются таким образом, чтобы с учетом выравнивания не было неиспользуемой памяти. + При генерации графа случайно задается значение, хранимое в вершине и номера соседних вершин, отдельным параметром генератора задется шанс получения номера соседней вершины больше или меньше номера данной вершины (соседняя вершина раньше или позже). + +- `ema-join-*`: Необходимо реализовать алгоритм equi-join для объединения + таблиц в файлах. На вход подаются два файла, результат их объединения пишется в + новый файл. В первой строке файла содержится количество строчек в файле. В + каждой последующей строчке входного файла содержится два значения, разделенных + пробелами: численный идентификатор и строка без пробелов фиксированного размера + 8 байт. Строка является валидным английским словом. Таблицы необходимо + объединять по идентификатору. + + По варианту выдается стратегия джоина: + - `ema-join-nl`: Nested Loop Join + - `ema-join-hash`: Hash Join Join + - `ema-join-sm`: Sort + Merge Join + + Количество строк входных таблиц включают все комбинации пар значений: 5, 10, 100, 1000, 10000." \ No newline at end of file diff --git a/lab/vtpc/vtld/ema_replace_str.c b/lab/vtpc/vtld/ema_replace_str.c new file mode 100644 index 0000000..a7abcc3 --- /dev/null +++ b/lab/vtpc/vtld/ema_replace_str.c @@ -0,0 +1,201 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ema_utils.h" +#include "util.h" +#include "vtpc.h" + +static void print_usage(const char* prog) { + fprintf( + stderr, + "Usage: %s --block_size N --file PATH --repeats N --string STR " + "--replace STR [--size N] [--range A-B] " + "[--type sequence/random]\n", + prog + ); +} + +int main(int argc, char** argv) { + const char* block_size_s = get_opt_value(argc, argv, "block_size"); + const char* file_s = get_opt_value(argc, argv, "file"); + const char* range_s = get_opt_value(argc, argv, "range"); + const char* type_s = get_opt_value(argc, argv, "type"); + const char* repeats_s = get_opt_value(argc, argv, "repeats"); + const char* string_s = get_opt_value(argc, argv, "string"); + const char* replace_s = get_opt_value(argc, argv, "replace"); + const char* size_s = get_opt_value(argc, argv, "size"); + + if (!block_size_s || !file_s || !repeats_s || !string_s || !replace_s) { + print_usage(argv[0]); + return 1; + } + + uint64_t block_size_u = 0; + uint64_t repeats = 0; + uint64_t gen_size = 0; + if (!parse_u64(block_size_s, &block_size_u) || + !parse_u64(repeats_s, &repeats)) { + fprintf(stderr, "Invalid block_size or repeats.\n"); + return 1; + } + if (size_s && !parse_u64(size_s, &gen_size)) { + fprintf(stderr, "Invalid size.\n"); + return 1; + } + if (block_size_u == 0 || repeats == 0) { + fprintf(stderr, "block_size and repeats must be > 0.\n"); + return 1; + } + + size_t block_size = (size_t)block_size_u; + size_t pattern_len = strlen(string_s); + if (pattern_len == 0 || strlen(replace_s) != pattern_len) { + fprintf(stderr, "string and replace must be non-empty and same length.\n"); + return 1; + } + if (pattern_len > block_size) { + fprintf(stderr, "string length must be <= block_size.\n"); + return 1; + } + + int random_mode = 0; + if (type_s) { + if (strcmp(type_s, "random") == 0) { + random_mode = 1; + } else if (strcmp(type_s, "sequence") != 0) { + fprintf(stderr, "Invalid type value: use sequence/random.\n"); + return 1; + } + } + + uint64_t range_start = 0; + uint64_t range_end = 0; + if (!parse_range(range_s, &range_start, &range_end)) { + fprintf(stderr, "Invalid range format, expected A-B.\n"); + return 1; + } + + srand((unsigned)time(NULL)); + + if (size_s) { + if (gen_size == 0) { + fprintf(stderr, "size must be > 0.\n"); + return 1; + } + if (!generate_file(file_s, gen_size, block_size, string_s)) { + return 1; + } + } + + int fd = vtpc_open(file_s, O_RDWR, 0); + if (fd < 0) { + perror("vtpc_open"); + return 1; + } + + off_t file_size_off = vtpc_lseek(fd, 0, SEEK_END); + if (file_size_off < 0) { + perror("vtpc_lseek"); + vtpc_close(fd); + return 1; + } + uint64_t file_size = (uint64_t)file_size_off; + if (file_size == 0) { + fprintf(stderr, "File is empty.\n"); + vtpc_close(fd); + return 1; + } + + if (range_start == 0 && range_end == 0) { + range_start = 0; + range_end = file_size; + } + if (range_end > file_size || range_start >= range_end) { + fprintf(stderr, "Invalid range bounds.\n"); + vtpc_close(fd); + return 1; + } + + uint64_t range_len = range_end - range_start; + uint64_t full_blocks = range_len / block_size; + uint64_t tail = range_len % block_size; + uint64_t blocks = full_blocks + ((tail > 0) ? 1 : 0); + if (blocks == 0) { + fprintf(stderr, "No blocks to process.\n"); + vtpc_close(fd); + return 1; + } + + void* buf = malloc(block_size); + if (!buf) { + fprintf(stderr, "malloc failed.\n"); + vtpc_close(fd); + return 1; + } + + uint64_t total_replaced = 0; + for (uint64_t i = 0; i < repeats; ++i) { + uint64_t block_index = + random_mode ? (uint64_t)(rand() % (int)blocks) : (i % blocks); + uint64_t offset = range_start + block_index * block_size; + size_t len = block_size; + if (tail > 0 && block_index == blocks - 1) { + len = (size_t)tail; + } + + if (vtpc_lseek(fd, (off_t)offset, SEEK_SET) == (off_t)-1) { + perror("vtpc_lseek"); + free(buf); + vtpc_close(fd); + return 1; + } + ssize_t r = vtpc_read(fd, buf, len); + if (r < 0) { + perror("vtpc_read"); + free(buf); + vtpc_close(fd); + return 1; + } + if (r == 0) { + continue; + } + size_t replaced = + replace_in_buffer((char*)buf, (size_t)r, string_s, replace_s); + if (replaced > 0) { + if (vtpc_lseek(fd, (off_t)offset, SEEK_SET) == (off_t)-1) { + perror("vtpc_lseek"); + free(buf); + vtpc_close(fd); + return 1; + } + ssize_t w = vtpc_write(fd, buf, (size_t)r); + if (w < 0 || w != r) { + perror("vtpc_write"); + free(buf); + vtpc_close(fd); + return 1; + } + total_replaced += replaced; + } + } + + printf("replacements=%" PRIu64 "\n", total_replaced); + + if (vtpc_fsync(fd) != 0) { + perror("vtpc_fsync"); + free(buf); + vtpc_close(fd); + return 1; + } + + free(buf); + vtpc_close(fd); + return 0; +} diff --git a/lab/vtpc/vtld/ema_utils.c b/lab/vtpc/vtld/ema_utils.c new file mode 100644 index 0000000..3ff1273 --- /dev/null +++ b/lab/vtpc/vtld/ema_utils.c @@ -0,0 +1,126 @@ +#include "ema_utils.h" + +#include +#include +#include +#include +#include +#include + +int parse_u64(const char* s, uint64_t* out) { + if (!s || !*s) { + return 0; + } + char* end = NULL; + errno = 0; + unsigned long long v = strtoull(s, &end, 10); + if (errno != 0 || !end || *end != '\0') { + return 0; + } + *out = (uint64_t)v; + return 1; +} + +int parse_range(const char* s, uint64_t* start, uint64_t* end) { + if (!s) { + *start = 0; + *end = 0; + return 1; + } + const char* dash = strchr(s, '-'); + if (!dash) { + return 0; + } + char left[32]; + char right[32]; + size_t left_len = (size_t)(dash - s); + size_t right_len = strlen(dash + 1); + if (left_len == 0 || right_len == 0 || left_len >= sizeof(left) || + right_len >= sizeof(right)) { + return 0; + } + memcpy(left, s, left_len); + left[left_len] = '\0'; + memcpy(right, dash + 1, right_len); + right[right_len] = '\0'; + if (!parse_u64(left, start) || !parse_u64(right, end)) { + return 0; + } + return 1; +} + +int open_file_direct(const char* path, int direct) { + int flags = O_RDWR; +#ifdef O_DIRECT + if (direct) { + flags |= O_DIRECT; + } +#else + if (direct) { + fprintf(stderr, "Warning: O_DIRECT is not supported, ignoring.\n"); + } +#endif + return open(path, flags); +} + +int generate_file( + const char* path, uint64_t size, size_t block_size, const char* pattern +) { + int fd = open(path, O_CREAT | O_TRUNC | O_RDWR, 0644); + if (fd < 0) { + perror("open"); + return 0; + } + size_t pat_len = strlen(pattern); + char* buf = malloc(block_size); + if (!buf) { + fprintf(stderr, "malloc failed.\n"); + close(fd); + return 0; + } + uint64_t remaining = size; + while (remaining > 0) { + size_t chunk = (remaining > block_size) ? block_size : (size_t)remaining; + for (size_t i = 0; i < chunk; ++i) { + buf[i] = (char)('a' + (rand() % 26)); + } + if (pat_len > 0 && pat_len <= chunk && (rand() % 3) == 0) { + size_t pos = (size_t)(rand() % (chunk - pat_len + 1)); + memcpy(buf + pos, pattern, pat_len); + } + ssize_t w = write(fd, buf, chunk); + if (w < 0 || (size_t)w != chunk) { + perror("write"); + free(buf); + close(fd); + return 0; + } + remaining -= chunk; + } + free(buf); + if (fsync(fd) != 0) { + perror("fsync"); + close(fd); + return 0; + } + close(fd); + return 1; +} + +size_t replace_in_buffer( + char* buf, size_t len, const char* needle, const char* repl +) { + size_t nlen = strlen(needle); + if (nlen == 0 || nlen > len) { + return 0; + } + size_t count = 0; + for (size_t i = 0; i + nlen <= len; ++i) { + if (memcmp(buf + i, needle, nlen) == 0) { + memcpy(buf + i, repl, nlen); + ++count; + i += nlen - 1; + } + } + return count; +} diff --git a/lab/vtpc/vtld/ema_utils.h b/lab/vtpc/vtld/ema_utils.h new file mode 100644 index 0000000..f33af3e --- /dev/null +++ b/lab/vtpc/vtld/ema_utils.h @@ -0,0 +1,14 @@ +#pragma once + +#include +#include + +int parse_u64(const char* s, uint64_t* out); +int parse_range(const char* s, uint64_t* start, uint64_t* end); +int open_file_direct(const char* path, int direct); +int generate_file( + const char* path, uint64_t size, size_t block_size, const char* pattern +); +size_t replace_in_buffer( + char* buf, size_t len, const char* needle, const char* repl +); diff --git a/lab/vtpc/vtld/makefile b/lab/vtpc/vtld/makefile new file mode 100644 index 0000000..a541767 --- /dev/null +++ b/lab/vtpc/vtld/makefile @@ -0,0 +1,10 @@ +CC=gcc +CFLAGS=-O3 -D_GNU_SOURCE -I../lib + +all: ema_replace_str + +ema_replace_str: ema_replace_str.c ema_utils.c + $(CC) $(CFLAGS) -o ema_replace_str ema_replace_str.c ema_utils.c util.c ../lib/vtpc.c ../lib/vtpc_state.c ../lib/vtpc_list.c ../lib/vtpc_cache.c + +clean: + rm -f ema_replace_str diff --git a/lab/vtpc/vtld/util.c b/lab/vtpc/vtld/util.c new file mode 100644 index 0000000..d0839ba --- /dev/null +++ b/lab/vtpc/vtld/util.c @@ -0,0 +1,21 @@ +#include "util.h" + +const char* get_opt_value(int argc, char** argv, const char* key) { + size_t key_len = strlen(key); + for (int i = 1; i < argc; ++i) { + const char* arg = argv[i]; + while (*arg == '-') { + ++arg; + } + const char* eq = strchr(arg, '='); + if (eq) { + size_t len = (size_t)(eq - arg); + if (len == key_len && strncmp(arg, key, len) == 0) { + return eq + 1; + } + } else if (strcmp(arg, key) == 0 && i + 1 < argc) { + return argv[i + 1]; + } + } + return NULL; +} diff --git a/lab/vtpc/vtld/util.h b/lab/vtpc/vtld/util.h new file mode 100644 index 0000000..4efea74 --- /dev/null +++ b/lab/vtpc/vtld/util.h @@ -0,0 +1,6 @@ +#pragma once + +#include +#include + +const char* get_opt_value(int argc, char** argv, const char* key); From 3b0ea469ff92523eef135b48dd4d4a61b9548c46 Mon Sep 17 00:00:00 2001 From: Ivan Zinchenko Date: Sat, 14 Feb 2026 03:24:17 +0300 Subject: [PATCH 7/7] WIP --- lab/vtpc/lib/CMakeLists.txt | 6 ------ lab/vtpc/lib/vtpc_cache.c | 5 +++++ lab/vtpc/lib/vtpc_internal.h | 9 +++++++-- lab/vtpc/lib/vtpc_state.c | 17 +++++++++++++++++ 4 files changed, 29 insertions(+), 8 deletions(-) diff --git a/lab/vtpc/lib/CMakeLists.txt b/lab/vtpc/lib/CMakeLists.txt index f5d3ab3..d565ca5 100644 --- a/lab/vtpc/lib/CMakeLists.txt +++ b/lab/vtpc/lib/CMakeLists.txt @@ -12,9 +12,3 @@ target_include_directories( PUBLIC . ) - -target_compile_definitions( - vtpc - PRIVATE - _GNU_SOURCE -) diff --git a/lab/vtpc/lib/vtpc_cache.c b/lab/vtpc/lib/vtpc_cache.c index 222c415..7acef97 100644 --- a/lab/vtpc/lib/vtpc_cache.c +++ b/lab/vtpc/lib/vtpc_cache.c @@ -205,8 +205,11 @@ void vtpc_move_resident_to_am(int idx) { int vtpc_get_or_load_page( int file_slot, off_t page_no, int need_existing, int* out_idx ) { + ++g_state.cache_all; + int resident_idx = vtpc_find_resident_page(file_slot, page_no); if (resident_idx >= 0) { + ++g_state.cache_hit; if (g_state.resident[resident_idx].list == VTPC_LIST_A1IN) { vtpc_move_resident_to_am(resident_idx); } else if (g_state.resident[resident_idx].list == VTPC_LIST_AM) { @@ -217,6 +220,8 @@ int vtpc_get_or_load_page( return 0; } + ++g_state.cache_miss; + if (vtpc_ensure_resident_capacity() < 0) { return -1; } diff --git a/lab/vtpc/lib/vtpc_internal.h b/lab/vtpc/lib/vtpc_internal.h index 64e98a0..5fcba99 100644 --- a/lab/vtpc/lib/vtpc_internal.h +++ b/lab/vtpc/lib/vtpc_internal.h @@ -1,11 +1,12 @@ #pragma once #include +#include #include #define VTPC_MAX_OPEN_FILES 256 -#define VTPC_MAX_CACHE_PAGES 128 -#define VTPC_MAX_A1OUT_PAGES 128 +#define VTPC_MAX_CACHE_PAGES 1024 +#define VTPC_MAX_A1OUT_PAGES 1024 enum vtpc_resident_list { VTPC_LIST_NONE = 0, @@ -65,6 +66,10 @@ typedef struct { vtpc_list am; vtpc_list a1out; + uint64_t cache_hit; + uint64_t cache_miss; + uint64_t cache_all; + vtpc_file files[VTPC_MAX_OPEN_FILES]; vtpc_resident_page resident[VTPC_MAX_CACHE_PAGES]; vtpc_ghost_page ghosts[VTPC_MAX_A1OUT_PAGES]; diff --git a/lab/vtpc/lib/vtpc_state.c b/lab/vtpc/lib/vtpc_state.c index 29a5663..54588df 100644 --- a/lab/vtpc/lib/vtpc_state.c +++ b/lab/vtpc/lib/vtpc_state.c @@ -1,10 +1,22 @@ #include +#include +#include #include #include "vtpc_internal.h" vtpc_state g_state = {0}; +static void vtpc_print_stats(void) { + fprintf( + stderr, + "vtpc stats: cache_hit=%llu cache_miss=%llu cache_all=%llu\n", + (unsigned long long)g_state.cache_hit, + (unsigned long long)g_state.cache_miss, + (unsigned long long)g_state.cache_all + ); +} + size_t vtpc_min_size(size_t lhs, size_t rhs) { return lhs < rhs ? lhs : rhs; } @@ -34,6 +46,11 @@ int vtpc_state_init(void) { g_state.a1out.head = -1; g_state.a1out.tail = -1; + if (atexit(vtpc_print_stats) != 0) { + errno = EINVAL; + return -1; + } + g_state.initialized = 1; return 0; }