- Pacman-dev - lists.archlinux.org

[pacman-dev] [PATCH 1/2] Added a new database backend
by Sivert Berg 18 Dec '08

18 Dec '08

This is an attempt to add a one-file-per-database backend. The main reason I wrote this was because load times of the database on my laptop when it's not yet cached is around 40-50 seconds. When you just want to do a quick "pacman -Ss" that can be a little annoying. I know you have discussed the current database vs. one-file earlier, and I think someone said another backend could be supported too. This first patch contains the main database code and a program to convert .db.tar.gz and file databases into the new format. Signed-off-by: Sivert Berg <sivertb(a)stud.ntnu.no> --- lib/libalpm/be_packed.h | 757 +++++++++++++++++++++++++++++++++++++++++++++++ src/util/Makefile.am | 4 +- src/util/convdb.c | 236 +++++++++++++++ 3 files changed, 996 insertions(+), 1 deletions(-) create mode 100644 lib/libalpm/be_packed.h create mode 100644 src/util/convdb.c diff --git a/lib/libalpm/be_packed.h b/lib/libalpm/be_packed.h new file mode 100644 index 0000000..e118a3a --- /dev/null +++ b/lib/libalpm/be_packed.h @@ -0,0 +1,757 @@ +/* + * Copyright (c) 2008 Sivert Berg + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/* _HOW IT WORKS_ + * + * If you've ever looked at the ext2 filesystem you will find alot + * of similarities (among others the inode that I didn't have the + * imagination to find a new name for). + * + * _GROUP_ + * Basicly the database is divided into groups. Each group got the + * same number of blocks and inodes. + * + * _INODE_ + * Each file is assigned an unique inode. The inode got a list of + * blocks where the actual data for the file is stored. Worth noting + * is that the last entry in the block list is indirect. That means + * it points to a block that points to blocks. + */ + +#include <stdint.h> +#include <sys/mman.h> +#include <stdlib.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <stdio.h> +#include <string.h> +#include <unistd.h> + + +#ifndef BE_PACKED_H +#define BE_PACKED_H + +#define BLOCK_SIZE 256 +#define GROUP_SIZE (BLOCK_SIZE * 4096) +#define BLOCKS_PER_GROUP 3936 +#define INODES_PER_GROUP 1024 + +#define MIN(a, b) (((a) < (b))?(a):(b)) + +/* _pack_open_file flags */ +#define PO_READ 1 +#define PO_WRITE 2 + + +typedef struct __inode_t { + int32_t size; + int32_t block[7]; +} inode_t; + +typedef struct __group_desc_t { + uint32_t inodes_free; + uint32_t blocks_free; + uint32_t checksum; + inode_t i_table[INODES_PER_GROUP]; + uint32_t bitmap[BLOCKS_PER_GROUP / 32]; + char data[]; +} group_desc_t; + +typedef struct __package_t { + int32_t inode; + uint16_t rec_len; + uint16_t name_len; + char name[]; +} package_t; + +typedef struct __pack_db_t { + int fd; + int groups; + int read_only; + group_desc_t *map; +} pack_db_t; + +typedef struct __pack_file_t { + int inode; + size_t pos; +} pack_file_t; + +typedef struct __pack_dir_t { + size_t pos; +} pack_dir_t; + + +/** + ** GROUP FUNCTIONS + */ +/* Initilizes a new group */ +void _pack_init_group(group_desc_t *group) +{ + int i, j; + + for (i = 0; i < INODES_PER_GROUP; i++) { + group->i_table[i].size = -1; + for (j = 0; j < 7; j++) + group->i_table[i].block[j] = -1; + } + + for (i = 0; i < BLOCKS_PER_GROUP / 32; i++) { + group->bitmap[i] = 0; + } + + group->inodes_free = INODES_PER_GROUP; + group->blocks_free = BLOCKS_PER_GROUP; + group->checksum = 0; +} + + +/* Adds a new group to the file */ +void _pack_new_group(pack_db_t *db) +{ + db->groups++; + ftruncate(db->fd, db->groups * GROUP_SIZE); + db->map = mremap(db->map, (db->groups - 1) * GROUP_SIZE, + db->groups * GROUP_SIZE, MREMAP_MAYMOVE); + + _pack_init_group((group_desc_t*)((char*)db->map + + GROUP_SIZE * (db->groups - 1))); +} + + +/** + ** POINTER FUNCTIONS (return pointers to various structures) + */ +group_desc_t *_pack_get_group(pack_db_t *db, int group) +{ + return (group_desc_t*)((char*)db->map + group*GROUP_SIZE); +} + + +inode_t *_pack_get_inode(pack_db_t *db, int inode) +{ + int group = inode / INODES_PER_GROUP; + group_desc_t *groupd = _pack_get_group(db, group); + inode %= INODES_PER_GROUP; + + return groupd->i_table + inode; +} + +void *_pack_get_block(pack_db_t *db, int block) +{ + if (block == -1) + return NULL; + + group_desc_t *group = _pack_get_group(db, block / BLOCKS_PER_GROUP); + return group->data + (block % BLOCKS_PER_GROUP) * BLOCK_SIZE; +} + + +/* Searches for a free block in the bitmap and marks the bit + * it finds as used. + */ +int _pack_check_bitmap(group_desc_t *group) +{ + int i = 0, j = 0, t = 0; + + for (i = 0; i < BLOCKS_PER_GROUP / 32; i++) { + if (group->bitmap[i] != 0xffffffff) { + t = group->bitmap[i]; + for (j = 0; j < 32 && (t & 1); j++, t >>= 1); + break; + } + } + + group->bitmap[i] |= 1 << j; + return i * 32 + j; +} + + +/* Clears the n'th bit */ +int _pack_bitmap_free(group_desc_t *group, int n) +{ + int i = n / 32; + int j = n % 32; + + group->bitmap[i] &= ~(i << j); + return 0; +} + + +/* Returns the number of a new unused block + * CAUTION: THIS FUNCTION MAY RESULT IN A MMREMAP! + * that means ANY pointers into the database may be invalid + * after you've called this function. + */ +int _pack_get_free_block(pack_db_t *db) +{ + int i; + group_desc_t *group; + int ret; + + for (i = 0; i < db->groups; i++) { + group = _pack_get_group(db, i); + if (group->blocks_free) { + group->blocks_free--; + ret = _pack_check_bitmap(group); + ret += i * BLOCKS_PER_GROUP; + return ret;; + } + } + + _pack_new_group(db); + group = _pack_get_group(db, i); + group->blocks_free--; + ret = _pack_check_bitmap(group); + ret += i * BLOCKS_PER_GROUP; + + return ret; +} + + +/* Frees the block given */ +int _pack_block_free(pack_db_t *db, int block) +{ + group_desc_t *group; + int g, b; + + g = block / BLOCKS_PER_GROUP; + b = block & BLOCKS_PER_GROUP; + + group = _pack_get_group(db, g); + group->blocks_free++; + + return _pack_bitmap_free(group, b); +} + + +/* Returns the number of a new free inode. + * CAUTION: THIS FUNCTION MAY RESULT IN A MMREMAP! + * that means ANY pointers into the database may be invalid + * after you've called this function. + */ +int _pack_get_free_inode(pack_db_t *db) +{ + int i = 0, j = 0; + group_desc_t *group; + + for (i = 0; i < db->groups; i++) { + group = _pack_get_group(db, i); + if (group->inodes_free) { + group->inodes_free--; + for (j = 0; j < INODES_PER_GROUP; j++) { + if (group->i_table[j].size == -1) { + group->i_table[j].size = 0; + return i * INODES_PER_GROUP + j; + } + } + } + } + + _pack_new_group(db); + group = _pack_get_group(db, i); + group->inodes_free--; + + return i * INODES_PER_GROUP; +} + + +/* Frees an inode and all of the blocks it might be using */ +int _pack_inode_free(pack_db_t *db, int in) +{ + inode_t *inode = _pack_get_inode(db, in); + group_desc_t *group = _pack_get_group(db, in / INODES_PER_GROUP); + int i; + int block; + int32_t *blocks; + int size = BLOCK_SIZE / sizeof(uint32_t) - 1; + + inode->size = -1; + + for (i = 0; i < 6; i++) { + if (inode->block[i] != -1) { + _pack_block_free(db, inode->block[i]); + inode->block[i] = -1; + } + } + + block = inode->block[6]; + blocks = _pack_get_block(db, block); + + while(blocks != NULL) { + for (i = 0; i < size; i++) { + if (blocks[i] != -1) + _pack_block_free(db, blocks[i]); + } + + _pack_block_free(db, block); + block = blocks[size]; + blocks = _pack_get_block(db, block); + } + + group->inodes_free++; + + return 0; +} + + +/* Returns the number of the block associated with the inode */ +int _pack_inode_get_block(pack_db_t *db, int i, int block) +{ + inode_t *inode = _pack_get_inode(db, i); + if (block < 6) + return inode->block[block]; + + block -= 6; + int size = BLOCK_SIZE / sizeof(uint32_t) - 1; + uint32_t *blocks = _pack_get_block(db, inode->block[6]); + + for (;blocks != NULL && block >= size; block -= size) + blocks = _pack_get_block(db, blocks[size]); + + if (blocks == NULL) + return -1; + + return blocks[block]; +} + + +/* Returns the number of the block block associated with the inode. + * If there is no such block, it gets allocated. + * CAUTION: THIS FUNCTION MAY RESULT IN A MMREMAP! + * that means ANY pointers into the database may be invalid + * after you've called this function. + */ +int _pack_inode_add_block(pack_db_t *db, int in, int block) +{ + inode_t *inode = _pack_get_inode(db, in); + int tmp; + + if (block < 6) { + if (inode->block[block] != -1) + return inode->block[block]; + + tmp = _pack_get_free_block(db); + inode = _pack_get_inode(db, in); + return inode->block[block] = tmp; + } + + block -= 6; + int size = BLOCK_SIZE / sizeof(int32_t) - 1; + int i; + int old_block; + int32_t *blocks; + + if (inode->block[6] == -1) { + tmp = _pack_get_free_block(db); + inode = _pack_get_inode(db, in); + inode->block[6] = tmp; + blocks = _pack_get_block(db, inode->block[6]); + for (i = 0; i < size + 1; i++) + blocks[i] = -1; + } + else + blocks = _pack_get_block(db, inode->block[6]); + + old_block = inode->block[6]; + + for (;block >= size; block -= size) { + if (blocks[size] == -1) { + tmp = _pack_get_free_block(db); + blocks = _pack_get_block(db, old_block); + blocks[size] = tmp; + old_block = tmp; + blocks = _pack_get_block(db, tmp); + for (i = 0; i < size + 1; i++) + blocks[i] = -1; + } + else { + old_block = blocks[size]; + blocks = _pack_get_block(db, blocks[size]); + } + } + + if (blocks[block] == -1) { + tmp = _pack_get_free_block(db); + blocks = _pack_get_block(db, old_block); + blocks[block] = tmp; + } + + return blocks[block]; +} + +/* Returns a pointer to the package with the name pkgname. + * Returns NULL if it's not found + */ +package_t *_pack_find_package(pack_db_t *db, const char *pkgname) +{ + inode_t *inode = _pack_get_inode(db, 0); + int s_len = strlen(pkgname); + package_t *pkg; + int block = 0; + int b_pos = 0; + + pkg = _pack_get_block(db, _pack_inode_get_block(db, 0, block)); + + while (block < (inode->size / BLOCK_SIZE)) { + if (pkg->inode != 0 && + pkg->name_len == s_len && + strncmp(pkgname, pkg->name, s_len) == 0) + break; + + b_pos += pkg->rec_len; + pkg = (package_t*)((char*)pkg + pkg->rec_len); + if (b_pos >= BLOCK_SIZE) { + pkg = _pack_get_block(db, _pack_inode_get_block(db, 0, ++block)); + b_pos = 0; + } + } + + return pkg; +} + +/* Creates a new package with the name pkgname. + * Returns a pointer to the new package. + * CAUTION: THIS FUNCTION MAY RESULT IN A MMREMAP! + * that means ANY pointers into the database may be invalid + * after you've called this function. + */ +package_t *_pack_add_package(pack_db_t *db, const char *pkgname) +{ + inode_t *inode = _pack_get_inode(db, 0); + int s_len = strlen(pkgname); + int len = s_len + sizeof(package_t); + package_t *pkg; + int block = 0; + int b_pos = 0; + + pkg = _pack_get_block(db, _pack_inode_get_block(db, 0, block)); + + while (pkg && block < (inode->size / BLOCK_SIZE)) { + if (pkg->rec_len >= len && pkg->inode == 0) + break; + + b_pos += pkg->rec_len; + pkg = (package_t*)((char*)pkg + pkg->rec_len); + if (b_pos >= BLOCK_SIZE) { + pkg = _pack_get_block(db, _pack_inode_get_block(db, 0, ++block)); + b_pos = 0; + } + } + + if (pkg == NULL) { + pkg = _pack_get_block(db, _pack_inode_add_block(db, 0, block)); + pkg->rec_len = BLOCK_SIZE; + pkg->inode = 0; + inode = _pack_get_inode(db, 0); + inode->size += BLOCK_SIZE; + } + + strncpy(pkg->name, pkgname, s_len); + if (pkg->rec_len > (len + sizeof(package_t) + 10)) { + package_t *foo = (package_t*)((char*)pkg + len); + foo->rec_len = pkg->rec_len - len; + foo->inode = 0; + pkg->rec_len = len; + } + + pkg->inode = -1; + pkg->name_len = s_len; + + return pkg; +} + + +/* Reads data from the file until buf is full or we hit a newline */ +int _pack_file_gets(pack_db_t *db, pack_file_t *file, char *buf, size_t size) +{ + inode_t *inode = _pack_get_inode(db, file->inode); + size_t i; + size_t b_pos = file->pos % BLOCK_SIZE; + const char *src = _pack_get_block(db, + _pack_inode_get_block(db, file->inode, file->pos / BLOCK_SIZE)); + int newline = 0; + + for (i = 0 + ;i < size && file->pos < inode->size && !newline + ;i++, b_pos++, file->pos++) { + if (b_pos >= BLOCK_SIZE) { + src = _pack_get_block(db, + _pack_inode_get_block(db, file->inode, file->pos / BLOCK_SIZE)); + b_pos = 0; + } + + buf[i] = src[b_pos]; + + if (src[b_pos] == '\n') + newline = 1; + } + + if (i < size - 1) + buf[i] = '\0'; + + return i; +} + + +/* Write the data in buf into the file + * CAUTION: THIS FUNCTION MAY RESULT IN A MMREMAP! + * that means ANY pointers into the database may be invalid + * after you've called this function. + */ +int _pack_file_puts(pack_db_t *db, pack_file_t *file, + const char *buf, size_t size) +{ + size_t i; + size_t b_pos = file->pos % BLOCK_SIZE; + char *dest = _pack_get_block(db, + _pack_inode_add_block(db, file->inode, file->pos / BLOCK_SIZE)); + int block; + + for (i = 0; i < size; i++, file->pos++, b_pos++) + { + if (b_pos >= BLOCK_SIZE) { + block = _pack_inode_add_block(db, file->inode, file->pos / BLOCK_SIZE); + dest = _pack_get_block(db, block); + b_pos = 0; + } + dest[b_pos] = buf[i]; + } + + inode_t *inode = _pack_get_inode(db, file->inode); + inode->size = file->pos; + + return i; +} + + +int _pack_file_eof(pack_db_t *db, pack_file_t *file) +{ + inode_t *inode = _pack_get_inode(db, file->inode); + return file->pos >= inode->size; +} + + +/* Opens a file in the database + * flags: if you pass PO_READ, it will return -1 + * if the file does not exist, if you pass PO_WRITE + * the file will be created if it does not exist + * CAUTION: THIS FUNCTION MAY RESULT IN A MMREMAP! + * that means ANY pointers into the database may be invalid + * after you've called this function. + */ +pack_file_t *_pack_open_file(pack_db_t *db, const char *pkgname, + const char *filename, int flags) +{ + char name[512]; + + snprintf(name, 512, "%s/%s", pkgname, filename); + + package_t *pkg = _pack_find_package(db, name); + int w = flags & PO_WRITE; + int tmp; + + if (w && db->read_only) + return NULL; + + if (pkg == NULL && w) + pkg = _pack_add_package(db, name); + else if (pkg == NULL) + return NULL; + + pack_file_t *ret = malloc(sizeof(pack_file_t)); + ret->pos = 0; + + if (pkg->inode == -1 && w) { + tmp = _pack_get_free_inode(db); + pkg = _pack_find_package(db, name); + ret->inode = pkg->inode = tmp; + } + else + ret->inode = pkg->inode; + + return ret; +} + + +/* Closes the file given */ +void _pack_close_file(pack_file_t *file) +{ + free(file); +} + + +/* Removes a package along with all it's files */ +int _pack_package_remove(pack_db_t *db, const char *pkgname) +{ + inode_t *inode = _pack_get_inode(db, 0); + int s_len = strlen(pkgname); + package_t *prev, *pkg; + int block = 0; + int b_pos = 0; + + if (db->read_only) + return -1; + + prev = NULL; + pkg = _pack_get_block(db, _pack_inode_get_block(db, 0, block)); + + while (block < (inode->size / BLOCK_SIZE)) { + if (pkg->inode != 0 && + pkg->name_len > s_len && + strncmp(pkgname, pkg->name, s_len) == 0 && + pkg->name[s_len] == '/') { + _pack_inode_free(db, pkg->inode); + + if (prev != NULL && prev->inode == 0) { + prev->rec_len += pkg->rec_len; + pkg = prev; + } + else + pkg->inode = 0; + } + + prev = pkg; + b_pos += pkg->rec_len; + pkg = (package_t*)((char*)pkg + pkg->rec_len); + if (b_pos >= BLOCK_SIZE) { + pkg = _pack_get_block(db, _pack_inode_get_block(db, 0, ++block)); + b_pos = 0; + prev = NULL; + } + } + + return 0; +} + + +/* Gets the next entry in the directory */ +int _pack_dir_next(pack_db_t *db, pack_dir_t *dir, char *name, size_t len) +{ + inode_t *inode = _pack_get_inode(db, 0); + + if (dir->pos >= inode->size) + return -1; + + int block, b_pos; + package_t *pkg; + + do { + block = dir->pos / BLOCK_SIZE; + b_pos = dir->pos % BLOCK_SIZE; + pkg = _pack_get_block(db, _pack_inode_get_block(db, 0, block)); + if (!pkg) + return -1; + pkg = (package_t*)((char*)pkg + b_pos); + dir->pos += pkg->rec_len; + } while (pkg->inode == 0); + + strncpy(name, pkg->name, MIN(len, pkg->name_len)); + if (len > pkg->name_len) + name[pkg->name_len] = '\0'; + + return 0; +} + + +/* Opens the directory for reading */ +pack_dir_t *_pack_open_dir(pack_db_t *db) +{ + pack_dir_t *dir = malloc(sizeof(pack_dir_t)); + dir->pos = 0; + + return dir; +} + + +/* Closes the given directory */ +void _pack_close_dir(pack_dir_t *dir) +{ + free(dir); +} + + +/* Opens a packed database */ +pack_db_t *_pack_db_open(const char *filename) +{ + struct stat st; + int is_new = 0; + + pack_db_t *db = malloc(sizeof(pack_db_t)); + db->fd = open(filename, O_RDWR, 0); + db->read_only = 0; + + if (db->fd == -1) { + db->fd = open(filename, O_RDONLY, 0); + if (db->fd == -1) { + db->fd = open(filename, O_RDWR | O_CREAT, 0644); + + if (db->fd == -1) { + free(db); + printf("Could not open %s\n", filename); + return NULL; + } + + ftruncate(db->fd, GROUP_SIZE); + is_new = 1; + } + else + db->read_only = 1; + } + + fstat(db->fd, &st); + db->groups = st.st_size / GROUP_SIZE; + + int flags = PROT_READ; + if (!db->read_only) + flags |= PROT_WRITE; + + db->map = mmap(NULL, GROUP_SIZE * db->groups, flags, MAP_SHARED, + db->fd, 0); + + if (db->map == MAP_FAILED) { + perror("mmap"); + free(db); + exit(0); + } + + if (is_new) { + _pack_init_group(db->map); + db->map->i_table[0].size = 0; + } + + return db; +} + + +/* Closes a packed database */ +void _pack_db_close(pack_db_t *db) +{ + munmap(db->map, db->groups * GROUP_SIZE); + close(db->fd); + free(db); +} + +#endif /* BE_PACKED_H */ diff --git a/src/util/Makefile.am b/src/util/Makefile.am index 97a0ffa..64e18ad 100644 --- a/src/util/Makefile.am +++ b/src/util/Makefile.am @@ -3,7 +3,7 @@ conffile = ${sysconfdir}/pacman.conf dbpath = ${localstatedir}/lib/pacman/ cachedir = ${localstatedir}/cache/pacman/pkg/ -bin_PROGRAMS = vercmp testpkg testdb +bin_PROGRAMS = vercmp testpkg testdb convdb DEFS = -DLOCALEDIR=\"@localedir@\" \ -DCONFFILE=\"$(conffile)\" \ @@ -18,6 +18,8 @@ AM_CFLAGS = -pedantic -D_GNU_SOURCE vercmp_SOURCES = vercmp.c vercmp_LDADD = $(top_builddir)/lib/libalpm/.libs/libalpm.la +convdb_SOURCES = convdb.c + testpkg_SOURCES = testpkg.c testpkg_LDADD = $(top_builddir)/lib/libalpm/.libs/libalpm.la diff --git a/src/util/convdb.c b/src/util/convdb.c new file mode 100644 index 0000000..f1f39bd --- /dev/null +++ b/src/util/convdb.c @@ -0,0 +1,236 @@ +/* + * Copyright (c) 2008 Sivert Berg + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + + +/* WARNING: This file is pretty much just a quick hack thrown together + * in a hurry. It seems to work, but don't take my word for it. + */ + +#include <be_packed.h> +#include <dirent.h> +#include <archive.h> +#include <archive_entry.h> + + +/* Convert an archieved database into a packed database + * Expects a gziped database ending in .db.tar.gz + */ +void convert_archive(const char *filename) +{ + struct archive_entry *aent = NULL; + struct archive *arch = archive_read_new(); + archive_read_support_compression_gzip(arch); + archive_read_support_format_tar(arch); + if (archive_read_open_filename(arch, filename, 1) != ARCHIVE_OK) { + printf("Could not open archive %s\n", archive_error_string(arch)); + return; + } + + char *db_name = strndup(filename, strlen(filename) - strlen(".tar.gz")); + pack_db_t *db = _pack_db_open(db_name); + free(db_name); + char buffer[4096]; + size_t size; + pack_file_t *pf; + + while (!archive_read_next_header(arch, &aent)) { + const char *type = NULL; + if (strstr(archive_entry_pathname(aent), "/desc")) + type = "desc"; + else if (strstr(archive_entry_pathname(aent), "/depends")) + type = "depends"; + else if (strstr(archive_entry_pathname(aent), "/install")) + type = "install"; + else if (strstr(archive_entry_pathname(aent), "/files")) + type = "files"; + + if (type == NULL) + continue; + + char *name = strdup(archive_entry_pathname(aent)); + int i; + for (i = 0; name[i] != '/' && name[i]; i++); + name[i] = '\0'; + + pf = _pack_open_file(db, name, type, PO_WRITE); + + free(name); + + do { + size = archive_read_data(arch, buffer, 4096); + _pack_file_puts(db, pf, buffer, size); + } while (size == 4096); + + _pack_close_file(pf); + + } + + _pack_db_close(db); + archive_read_close(arch); + archive_read_finish(arch); +} + + + +const char *dir_name(const char *full_path) +{ + const char *slash = full_path; + while (*full_path) + if (*full_path++ == '/') + slash = full_path; + return slash; +} + + +/* Convert a directory database into a packed database */ +void convert_directory(const char *pathname) +{ + char db_name[1024]; + pack_db_t *db; + DIR *dir = opendir(pathname); + struct dirent *dire; + char path[1024]; + char buffer[8196]; + size_t size; + FILE *f; + pack_file_t *pf; + + strcpy(db_name, dir_name(pathname)); + strcat(db_name, ".db"); + db = _pack_db_open(db_name); + + if (dir == NULL) { + perror("opendir"); + return; + } + + dire = readdir(dir); + while (dire != NULL) { + if (strcmp(dire->d_name, ".") && strcmp(dire->d_name, "..")) { + + + size_t foo = 0; + sprintf(path, "%s/%s/desc", pathname, dire->d_name); + f = fopen(path, "r"); + if (f == NULL) + goto depends; + pf = _pack_open_file(db, dire->d_name, "desc", PO_WRITE); + do { + size = fread(buffer, 1, 8196, f); + _pack_file_puts(db, pf, buffer, size); + foo += size; + } while (size == 8196); + _pack_close_file(pf); + fclose(f); +depends: + sprintf(path, "%s/%s/depends", pathname, dire->d_name); + f = fopen(path, "r"); + if (f == NULL) + goto files; + pf = _pack_open_file(db, dire->d_name, "depends", PO_WRITE); + do { + size = fread(buffer, 1, 8196, f); + _pack_file_puts(db, pf, buffer, size); + foo += size; + } while (size == 8196); + _pack_close_file(pf); + fclose(f); +files: + sprintf(path, "%s/%s/files", pathname, dire->d_name); + f = fopen(path, "r"); + if (f == NULL) + goto install; + pf = _pack_open_file(db, dire->d_name, "files", PO_WRITE); + do { + size = fread(buffer, 1, 8196, f); + _pack_file_puts(db, pf, buffer, size); + foo += size; + } while (size == 8196); + _pack_close_file(pf); + fclose(f); +install: + sprintf(path, "%s/%s/install", pathname, dire->d_name); + f = fopen(path, "r"); + if (f == NULL) + goto changelog; + pf = _pack_open_file(db, dire->d_name, "install", PO_WRITE); + do { + size = fread(buffer, 1, 8196, f); + _pack_file_puts(db, pf, buffer, size); + foo += size; + } while (size == 8196); + _pack_close_file(pf); + fclose(f); +changelog: + sprintf(path, "%s/%s/changelog", pathname, dire->d_name); + f = fopen(path, "r"); + if (f == NULL) + goto deltas; + pf = _pack_open_file(db, dire->d_name, "changelog", PO_WRITE); + do { + size = fread(buffer, 1, 8196, f); + _pack_file_puts(db, pf, buffer, size); + foo += size; + } while (size == 8196); + _pack_close_file(pf); + fclose(f); +deltas: + sprintf(path, "%s/%s/deltas", pathname, dire->d_name); + f = fopen(path, "r"); + if (f == NULL) + goto done; + pf = _pack_open_file(db, dire->d_name, "deltas", PO_WRITE); + do { + size = fread(buffer, 1, 8196, f); + _pack_file_puts(db, pf, buffer, size); + foo += size; + } while (size == 8196); + _pack_close_file(pf); + fclose(f); +done: + pf = NULL; + } + + dire = readdir(dir); + } + + closedir(dir); + _pack_db_close(db); +} + +int main(int argc, char *argv[]) +{ + if (argc < 2) { + printf("Usage: %s <dir/file>\n", argv[0]); + exit(0); + } + + if (strstr(argv[1], ".tar.gz") != NULL) + convert_archive(argv[1]); + else + convert_directory(argv[1]); + + return 0; +} -- 1.6.0.5

3 7

[pacman-dev] [patch]GPG verification
by Jatheendra 17 Dec '08

17 Dec '08

These patches will add VerifySig option to pacman.conf. VerifySig takes options Always, Optional or Never [repo-name] Server = ServerName VerifySig = Always Include = IncludePath >From 77be2c5cbfa3c7a750fe46d115c23096d2cf51e5 Mon Sep 17 00:00:00 2001 From: shankar <jatheendra(a)gmail.com> Date: Wed, 17 Dec 2008 20:52:21 +0530 Subject: [PATCH] changed gpg verification logic Signed-off-by: shankar <jatheendra(a)gmail.com> --- lib/libalpm/signing.c | 3 +++ lib/libalpm/sync.c | 26 ++++++++++++++++++++++---- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/lib/libalpm/signing.c b/lib/libalpm/signing.c index ddb89bc..0835b5e 100644 --- a/lib/libalpm/signing.c +++ b/lib/libalpm/signing.c @@ -166,6 +166,9 @@ pgpcheck_t _alpm_gpgme_checksig(const char *pkgpath, const pmpgpsig_t *sig) if(gpgsig->summary & GPGME_SIGSUM_VALID) { /* good signature, continue */ + ret = PM_PGP_SIG_VALID; + _alpm_log(PM_LOG_DEBUG, _("Package %s has a valid signature.\n"), + pkgpath); } else if(gpgsig->summary & GPGME_SIGSUM_GREEN) { /* 'green' signature, not sure what to do here */ _alpm_log(PM_LOG_WARNING, _("Package %s has a green signature.\n"), diff --git a/lib/libalpm/sync.c b/lib/libalpm/sync.c index 24f2b98..f658ae2 100644 --- a/lib/libalpm/sync.c +++ b/lib/libalpm/sync.c @@ -901,12 +901,30 @@ int _alpm_sync_commit(pmtrans_t *trans, pmdb_t *db_local, alpm_list_t **data) *data = alpm_list_add(*data, strdup(filename)); } /* check PGP signature next */ - if(_alpm_gpgme_checksig(filepath, pgpsig) == PM_PGP_SIG_INVALID) { - errors++; - *data = alpm_list_add(*data, strdup(filename)); + pmdb_t *sdb = alpm_pkg_get_db(spkg); + + if(sdb->verify_gpg == PM_GPG_VERIFY_ALWAYS) { + if(_alpm_gpgme_checksig(filepath, pgpsig) != PM_PGP_SIG_VALID) { + errors++; + *data = alpm_list_add(*data, strdup(filename)); + _alpm_log(PM_LOG_ERROR, _("Invalid GPG signature on package: %s\n"),alpm_pkg_get_name(spkg)); + } + FREE(filepath); + } else if (sdb->verify_gpg == PM_GPG_VERIFY_OPTIONAL) { + pgpcheck_t ret1 = _alpm_gpgme_checksig(filepath, pgpsig); + + if(ret1 == PM_PGP_SIG_MISSING) { + /*no problems here*/ + } else if (ret1 != PM_PGP_SIG_VALID) { + errors++; + *data = alpm_list_add(*data, strdup(filename)); + _alpm_log(PM_LOG_ERROR, _("Invalid GPG signature on package: %s\n"),alpm_pkg_get_name(spkg)); + } + FREE(filepath); } - FREE(filepath); } + + if(errors) { pm_errno = PM_ERR_PKG_INVALID; goto error; -- 1.6.0.4

1 0

[pacman-dev] [PATCH] Added a new database backend
by Sivert Berg 16 Dec '08

16 Dec '08

This is an attempt to add a one-file-per-database backend. This leads to significatly faster loading of the database when the database is not cached. As you can see from the patch minimal changes to the rest of ALPM was needed. The only change was to make reading of the changelog files happen through the backend. The database is basically a scaled down ext2-like filesystem. To make corruption detection possible in an early phase I have planned to add a checksum, but have not come that far yet. I have been using it on my laptop for a week or so now, and everything seems to be working as expected. I'm sorry it became such a large patch, but about half of it is mostly old code (be_packed.c is essentially be_files.c with fopen fprintf etc changed with new functions). Signed-off-by: Sivert Berg <sivertb(a)stud.ntnu.no> --- configure.ac | 7 + lib/libalpm/Makefile.am | 2 +- lib/libalpm/be_files.c | 21 ++ lib/libalpm/be_packed.c | 845 +++++++++++++++++++++++++++++++++++++++++++++++ lib/libalpm/be_packed.h | 757 ++++++++++++++++++++++++++++++++++++++++++ lib/libalpm/db.h | 3 + lib/libalpm/package.c | 12 +- src/util/Makefile.am | 4 +- src/util/convdb.c | 236 +++++++++++++ 9 files changed, 1876 insertions(+), 11 deletions(-) create mode 100644 lib/libalpm/be_packed.c create mode 100644 lib/libalpm/be_packed.h create mode 100644 src/util/convdb.c diff --git a/configure.ac b/configure.ac index c7841b8..6e28ac9 100644 --- a/configure.ac +++ b/configure.ac @@ -73,6 +73,12 @@ AC_ARG_WITH(root-dir, AS_HELP_STRING([--with-root-dir=path], [set the location of pacman's root operating directory]), [ROOTDIR=$withval], [ROOTDIR=/]) +# Help line for database backend +AC_ARG_WITH(backend, + AS_HELP_STRING([--with-backend=backend], [choose which backend to use. You can choose between 'files' and 'packed', it defaults to files]), + [BACKEND=$withval], [BACKEND=files]) +AC_SUBST(BACKEND) + # Help line for package extension AC_ARG_WITH(pkg-ext, AS_HELP_STRING([--with-pkg-ext=ext], [set the file extension used by packages]), @@ -362,6 +368,7 @@ ${PACKAGE_NAME}: package extension : ${PKGEXT} source pkg extension : ${SRCEXT} database extension : ${DBEXT} + database backend : ${BACKEND} Compilation options: Run make in doc/ dir : ${wantdoc} diff --git a/lib/libalpm/Makefile.am b/lib/libalpm/Makefile.am index 871855e..79b9d26 100644 --- a/lib/libalpm/Makefile.am +++ b/lib/libalpm/Makefile.am @@ -25,7 +25,7 @@ libalpm_la_SOURCES = \ alpm.h alpm.c \ alpm_list.h alpm_list.c \ backup.h backup.c \ - be_files.c \ + be_@BACKEND@.c \ be_package.c \ cache.h cache.c \ conflict.h conflict.c \ diff --git a/lib/libalpm/be_files.c b/lib/libalpm/be_files.c index b9ff646..81d7a17 100644 --- a/lib/libalpm/be_files.c +++ b/lib/libalpm/be_files.c @@ -874,4 +874,25 @@ int _alpm_db_remove(pmdb_t *db, pmpkg_t *info) return(ret); } +void *_alpm_db_changelog_open(pmdb_t *db, pmpkg_t *pkg) +{ + char clfile[PATH_MAX]; + snprintf(clfile, PATH_MAX, "%s/%s/%s-%s/changelog", + alpm_option_get_dbpath(), + alpm_db_get_name(handle->db_local), + alpm_pkg_get_name(pkg), + alpm_pkg_get_version(pkg)); + return fopen(clfile, "r"); +} + +int _alpm_db_changelog_close(pmdb_t *db, void *stream) +{ + return fclose(stream); +} + +size_t _alpm_db_changelog_read(pmdb_t *db, const void *pf, void *ptr, size_t size) +{ + return fread(ptr, 1, size, pf); +} + /* vim: set ts=2 sw=2 noet: */ diff --git a/lib/libalpm/be_packed.c b/lib/libalpm/be_packed.c new file mode 100644 index 0000000..3e4ea68 --- /dev/null +++ b/lib/libalpm/be_packed.c @@ -0,0 +1,845 @@ +/* + * be_packed.c + * + * Copyright (c) 2006 by Christian Hamar <krics(a)linuxforum.hu> + * Copyright (c) 2006 by Miklos Vajna <vmiklos(a)frugalware.org> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include "config.h" + +#include <unistd.h> +#include <stdio.h> +#include <stdlib.h> +#include <errno.h> +#include <string.h> +#include <stdint.h> /* uintmax_t, intmax_t */ +#include <sys/stat.h> +#include <dirent.h> +#include <ctype.h> +#include <time.h> +#include <utime.h> +#include <limits.h> /* PATH_MAX */ +#include <locale.h> /* setlocale */ + +/* libalpm */ +#include "db.h" +#include "alpm_list.h" +#include "cache.h" +#include "log.h" +#include "util.h" +#include "alpm.h" +#include "handle.h" +#include "package.h" +#include "delta.h" +#include "deps.h" +#include "dload.h" + +#include "be_packed.h" + +/** Update a package database + * @param force if true, then forces the update, otherwise update only in case + * the database isn't up to date + * @param db pointer to the package database to update + * @return 0 on success, > 0 on error (pm_errno is set accordingly), < 0 if up + * to date + */ +int SYMEXPORT alpm_db_update(int force, pmdb_t *db) +{ + char *dbfile, *dbfilepath; + time_t newmtime = 0; + char dbpath[PATH_MAX]; + char dbname[PATH_MAX]; + char buffer[1024]; + size_t len; + struct stat st; + struct utimbuf utb; + + int ret; + + ALPM_LOG_FUNC; + + /* Sanity checks */ + ASSERT(handle != NULL, RET_ERR(PM_ERR_HANDLE_NULL, -1)); + ASSERT(db != NULL && db != handle->db_local, RET_ERR(PM_ERR_WRONG_ARGS, -1)); + /* Verify we are in a transaction. This is done _mainly_ because we need a DB + * lock - if we update without a db lock, we may kludge some other pacman + * process that _has_ a lock. + */ + ASSERT(handle->trans != NULL, RET_ERR(PM_ERR_TRANS_NULL, -1)); + ASSERT(handle->trans->state == STATE_INITIALIZED, RET_ERR(PM_ERR_TRANS_NOT_INITIALIZED, -1)); + ASSERT(handle->trans->type == PM_TRANS_TYPE_SYNC, RET_ERR(PM_ERR_TRANS_TYPE, -1)); + + if(!alpm_list_find_ptr(handle->dbs_sync, db)) { + RET_ERR(PM_ERR_DB_NOT_FOUND, -1); + } + + strncpy(dbname, db->path, PATH_MAX - 3); + /* Strip of the trailing / */ + dbname[strlen(dbname) - 1] = '\0'; + strcat(dbname, ".db"); + stat(dbname, &st); + + if(!force) { + /* get the lastupdate time */ + if(st.st_mtime == 0) { + _alpm_log(PM_LOG_DEBUG, "failed to get last write time for %s\n", + db->treename); + } + } + else + st.st_mtime = 0; + + len = strlen(db->treename) + strlen(DBEXT) + 1; + MALLOC(dbfile, len, RET_ERR(PM_ERR_MEMORY, -1)); + sprintf(dbfile, "%s" DBEXT, db->treename); + + strncpy(dbpath, alpm_option_get_dbpath(), PATH_MAX); + strcpy(dbpath + strlen(dbpath), "sync/"); + + ret = _alpm_download_single_file(dbfile, db->servers, dbpath, + st.st_mtime, &newmtime); + free(dbfile); + + if(ret == 1) { + /* mtimes match, do nothing */ + pm_errno = 0; + return(1); + } else if(ret == -1) { + /* pm_errno was set by the download code */ + _alpm_log(PM_LOG_DEBUG, "failed to sync db: %s\n", alpm_strerrorlast()); + return(-1); + } else { + /* remove the old db */ + _alpm_log(PM_LOG_DEBUG, "removing database %s\n", db->path); + unlink(dbname); + + /* cache needs to be rebuilt */ + _alpm_db_free_pkgcache(db); + + /* form the path to the db location */ + len = strlen(dbpath) + strlen(db->treename) + strlen(DBEXT) + 1; + MALLOC(dbfilepath, len, RET_ERR(PM_ERR_MEMORY, -1)); + sprintf(dbfilepath, "%s%s" DBEXT, dbpath, db->treename); + + /* convert the database file */ + snprintf(buffer, 1024, "convdb %s", dbfilepath); + system(buffer); + + /* set the correct time on the new database */ + utb.actime = newmtime; + utb.modtime = newmtime; + utime(dbname, &utb); + + /* Reopen the database */ + _pack_db_close(db->handle); + db->handle = _pack_db_open(dbname); + + unlink(dbfilepath); + free(dbfilepath); + } + + return(0); +} + +int _alpm_db_open(pmdb_t *db) +{ + ALPM_LOG_FUNC; + char path[PATH_MAX]; + + if(db == NULL) { + RET_ERR(PM_ERR_DB_NULL, -1); + } + + strncpy(path, db->path, PATH_MAX - 3); + /* Strip of the trailing / */ + path[strlen(path) - 1] = '\0'; + strcat(path, ".db"); + + _alpm_log(PM_LOG_DEBUG, "opening database from path '%s'\n", db->path); + db->handle = _pack_db_open(path); + + if(db->handle == NULL) { + RET_ERR(PM_ERR_DB_OPEN, -1); + } + + return(0); +} + +void _alpm_db_close(pmdb_t *db) +{ + ALPM_LOG_FUNC; + + if(db == NULL) { + return; + } + + if(db->handle) { + _pack_db_close(db->handle); + db->handle = NULL; + } +} + +static int splitname(const char *target, pmpkg_t *pkg) +{ + /* the format of a db entry is as follows: + * package-version-rel/ + * package name can contain hyphens, so parse from the back- go back + * two hyphens and we have split the version from the name. + */ + char *tmp, *p, *q; + + if(target == NULL || pkg == NULL) { + return(-1); + } + STRDUP(tmp, target, RET_ERR(PM_ERR_MEMORY, -1)); + p = tmp + strlen(tmp); + + /* do the magic parsing- find the beginning of the version string + * by doing two iterations of same loop to lop off two hyphens */ + for(q = --p; *q && *q != '-'; q--); + for(p = --q; *p && *p != '-'; p--); + if(*p != '-' || p == tmp) { + return(-1); + } + + /* copy into fields and return */ + if(pkg->version) { + FREE(pkg->version); + } + STRDUP(pkg->version, p+1, RET_ERR(PM_ERR_MEMORY, -1)); + /* insert a terminator at the end of the name (on hyphen)- then copy it */ + *p = '\0'; + if(pkg->name) { + FREE(pkg->name); + } + STRDUP(pkg->name, tmp, RET_ERR(PM_ERR_MEMORY, -1)); + + free(tmp); + return(0); +} + +int _alpm_db_populate(pmdb_t *db) +{ + int count = 0; + int i; + char name[PATH_MAX]; + pack_dir_t *dir; + + ALPM_LOG_FUNC; + + ASSERT(db != NULL, RET_ERR(PM_ERR_DB_NULL, -1)); + + dir = _pack_open_dir(db->handle); + + while(_pack_dir_next(db->handle, dir, name, PATH_MAX) != -1) { + pmpkg_t *pkg; + + pkg = _alpm_pkg_new(); + if(pkg == NULL) { + return(-1); + } + + /* Deal only with desc entries, that way we're sure we do + * not add the same package twice + */ + if(strstr(name, "/desc") == NULL) + continue; + + for (i = 0; name[i] && name[i] != '/'; i++); + name[i] = '\0'; + + /* split the db entry name */ + if(splitname(name, pkg) != 0) { + _alpm_log(PM_LOG_ERROR, _("invalid name for database entry '%s'\n"), + name); + _alpm_pkg_free(pkg); + continue; + } + + /* explicitly read with only 'BASE' data, accessors will handle the rest */ + if(_alpm_db_read(db, pkg, INFRQ_BASE) == -1) { + _alpm_log(PM_LOG_ERROR, _("corrupted database entry '%s'\n"), name); + _alpm_pkg_free(pkg); + continue; + } + pkg->origin = PKG_FROM_CACHE; + pkg->origin_data.db = db; + /* add to the collection */ + _alpm_log(PM_LOG_FUNCTION, "adding '%s' to package cache for db '%s'\n", + pkg->name, db->treename); + db->pkgcache = alpm_list_add(db->pkgcache, pkg); + count++; + } + + db->pkgcache = alpm_list_msort(db->pkgcache, count, _alpm_pkg_cmp); + return(count); +} + +int _alpm_db_read(pmdb_t *db, pmpkg_t *info, pmdbinfrq_t inforeq) +{ + char line[513]; + char pkgname[PATH_MAX]; + pack_file_t *pf; + ALPM_LOG_FUNC; + + if(db == NULL) { + RET_ERR(PM_ERR_DB_NULL, -1); + } + + if(info == NULL || info->name == NULL || info->version == NULL) { + _alpm_log(PM_LOG_DEBUG, "invalid package entry provided to _alpm_db_read, skipping\n"); + return(-1); + } + + if(info->origin == PKG_FROM_FILE) { + _alpm_log(PM_LOG_DEBUG, "request to read database info for a file-based package '%s', skipping...\n", info->name); + return(-1); + } + + /* bitmask logic here: + * infolevel: 00001111 + * inforeq: 00010100 + * & result: 00000100 + * == to inforeq? nope, we need to load more info. */ + if((info->infolevel & inforeq) == inforeq) { + /* already loaded this info, do nothing */ + return(0); + } + _alpm_log(PM_LOG_FUNCTION, "loading package data for %s : level=0x%x\n", + info->name, inforeq); + + /* clear out 'line', to be certain - and to make valgrind happy */ + memset(line, 0, 513); + snprintf(pkgname, PATH_MAX, "%s-%s", info->name, info->version); + + /* DESC */ + if(inforeq & INFRQ_DESC) { + if((pf = _pack_open_file(db->handle, pkgname, "desc", PO_READ)) == NULL) { + _alpm_log(PM_LOG_ERROR, _("could not open package %s\n"), pkgname); + goto error; + } + while(!_pack_file_eof(db->handle, pf)) { + if(_pack_file_gets(db->handle, pf, line, 256) == 0) { + break; + } + _alpm_strtrim(line); + if(strcmp(line, "%NAME%") == 0) { + if(_pack_file_gets(db->handle, pf, line, 512) == 0) { + goto error; + } + if(strcmp(_alpm_strtrim(line), info->name) != 0) { + _alpm_log(PM_LOG_ERROR, _("%s database is inconsistent: name " + "mismatch on package %s %s\n"), db->treename, info->name, line); + } + } else if(strcmp(line, "%VERSION%") == 0) { + if(_pack_file_gets(db->handle, pf, line, 512) == 0) { + goto error; + } + if(strcmp(_alpm_strtrim(line), info->version) != 0) { + _alpm_log(PM_LOG_ERROR, _("%s database is inconsistent: version " + "mismatch on package %s\n"), db->treename, info->name); + } + } else if(strcmp(line, "%FILENAME%") == 0) { + if(_pack_file_gets(db->handle, pf, line, 512) == 0) { + goto error; + } + STRDUP(info->filename, _alpm_strtrim(line), goto error); + } else if(strcmp(line, "%DESC%") == 0) { + if(_pack_file_gets(db->handle, pf, line, 512) == 0) { + goto error; + } + STRDUP(info->desc, _alpm_strtrim(line), goto error); + } else if(strcmp(line, "%GROUPS%") == 0) { + while(_pack_file_gets(db->handle, pf, line, 512) && + strlen(_alpm_strtrim(line))) { + char *linedup; + STRDUP(linedup, _alpm_strtrim(line), goto error); + info->groups = alpm_list_add(info->groups, linedup); + } + } else if(strcmp(line, "%URL%") == 0) { + if(_pack_file_gets(db->handle, pf, line, 512) == 0) { + goto error; + } + STRDUP(info->url, _alpm_strtrim(line), goto error); + } else if(strcmp(line, "%LICENSE%") == 0) { + while(_pack_file_gets(db->handle, pf, line, 512) && + strlen(_alpm_strtrim(line))) { + char *linedup; + STRDUP(linedup, _alpm_strtrim(line), goto error); + info->licenses = alpm_list_add(info->licenses, linedup); + } + } else if(strcmp(line, "%ARCH%") == 0) { + if(_pack_file_gets(db->handle, pf, line, 512) == 0) { + goto error; + } + STRDUP(info->arch, _alpm_strtrim(line), goto error); + } else if(strcmp(line, "%BUILDDATE%") == 0) { + if(_pack_file_gets(db->handle, pf, line, 512) == 0) { + goto error; + } + _alpm_strtrim(line); + + char first = tolower(line[0]); + if(first > 'a' && first < 'z') { + struct tm tmp_tm = {0}; //initialize to null incase of failure + setlocale(LC_TIME, "C"); + strptime(line, "%a %b %e %H:%M:%S %Y", &tmp_tm); + info->builddate = mktime(&tmp_tm); + setlocale(LC_TIME, ""); + } else { + info->builddate = atol(line); + } + } else if(strcmp(line, "%INSTALLDATE%") == 0) { + if(_pack_file_gets(db->handle, pf, line, 512) == 0) { + goto error; + } + _alpm_strtrim(line); + + char first = tolower(line[0]); + if(first > 'a' && first < 'z') { + struct tm tmp_tm = {0}; //initialize to null incase of failure + setlocale(LC_TIME, "C"); + strptime(line, "%a %b %e %H:%M:%S %Y", &tmp_tm); + info->installdate = mktime(&tmp_tm); + setlocale(LC_TIME, ""); + } else { + info->installdate = atol(line); + } + } else if(strcmp(line, "%PACKAGER%") == 0) { + if(_pack_file_gets(db->handle, pf, line, 512) == 0) { + goto error; + } + STRDUP(info->packager, _alpm_strtrim(line), goto error); + } else if(strcmp(line, "%REASON%") == 0) { + if(_pack_file_gets(db->handle, pf, line, 512) == 0) { + goto error; + } + info->reason = atol(_alpm_strtrim(line)); + } else if(strcmp(line, "%SIZE%") == 0 || strcmp(line, "%CSIZE%") == 0) { + /* NOTE: the CSIZE and SIZE fields both share the "size" field + * in the pkginfo_t struct. This can be done b/c CSIZE + * is currently only used in sync databases, and SIZE is + * only used in local databases. + */ + if(_pack_file_gets(db->handle, pf, line, 512) == 0) { + goto error; + } + info->size = atol(_alpm_strtrim(line)); + /* also store this value to isize if isize is unset */ + if(info->isize == 0) { + info->isize = info->size; + } + } else if(strcmp(line, "%ISIZE%") == 0) { + /* ISIZE (installed size) tag only appears in sync repositories, + * not the local one. */ + if(_pack_file_gets(db->handle, pf, line, 512) == 0) { + goto error; + } + info->isize = atol(_alpm_strtrim(line)); + } else if(strcmp(line, "%MD5SUM%") == 0) { + /* MD5SUM tag only appears in sync repositories, + * not the local one. */ + if(_pack_file_gets(db->handle, pf, line, 512) == 0) { + goto error; + } + STRDUP(info->md5sum, _alpm_strtrim(line), goto error); + } else if(strcmp(line, "%REPLACES%") == 0) { + while(_pack_file_gets(db->handle, pf, line, 512) && + strlen(_alpm_strtrim(line))) { + char *linedup; + STRDUP(linedup, _alpm_strtrim(line), goto error); + info->replaces = alpm_list_add(info->replaces, linedup); + } + } else if(strcmp(line, "%FORCE%") == 0) { + info->force = 1; + } + } + _pack_close_file(pf); + pf = NULL; + } + + /* FILES */ + if(inforeq & INFRQ_FILES) { + if((pf = _pack_open_file(db->handle, pkgname, "files", PO_READ)) == NULL) { + _alpm_log(PM_LOG_ERROR, _("could not open package %s\n"), pkgname); + goto error; + } + while(_pack_file_gets(db->handle, pf, line, 256) && + strlen(_alpm_strtrim(line))) { + _alpm_strtrim(line); + if(strcmp(line, "%FILES%") == 0) { + while(_pack_file_gets(db->handle, pf, line, 512) && + strlen(_alpm_strtrim(line))) { + char *linedup; + STRDUP(linedup, _alpm_strtrim(line), goto error); + info->files = alpm_list_add(info->files, linedup); + } + } else if(strcmp(line, "%BACKUP%") == 0) { + while(_pack_file_gets(db->handle, pf, line, 512) && + strlen(_alpm_strtrim(line))) { + char *linedup; + STRDUP(linedup, _alpm_strtrim(line), goto error); + info->backup = alpm_list_add(info->backup, linedup); + } + } + } + _pack_close_file(pf); + pf = NULL; + } + + /* DEPENDS */ + if(inforeq & INFRQ_DEPENDS) { + if((pf = _pack_open_file(db->handle, pkgname, "depends", PO_READ)) == NULL) { + _alpm_log(PM_LOG_ERROR, _("could not open package %s\n"), pkgname); + goto error; + } + while(!_pack_file_eof(db->handle, pf)) { + _pack_file_gets(db->handle, pf, line, 255); + _alpm_strtrim(line); + if(strcmp(line, "%DEPENDS%") == 0) { + while(_pack_file_gets(db->handle, pf, line, 512) && + strlen(_alpm_strtrim(line))) { + pmdepend_t *dep = _alpm_splitdep(_alpm_strtrim(line)); + info->depends = alpm_list_add(info->depends, dep); + } + } else if(strcmp(line, "%OPTDEPENDS%") == 0) { + while(_pack_file_gets(db->handle, pf, line, 512) && + strlen(_alpm_strtrim(line))) { + char *linedup; + STRDUP(linedup, _alpm_strtrim(line), goto error); + info->optdepends = alpm_list_add(info->optdepends, linedup); + } + } else if(strcmp(line, "%CONFLICTS%") == 0) { + while(_pack_file_gets(db->handle, pf, line, 512) && + strlen(_alpm_strtrim(line))) { + char *linedup; + STRDUP(linedup, _alpm_strtrim(line), goto error); + info->conflicts = alpm_list_add(info->conflicts, linedup); + } + } else if(strcmp(line, "%PROVIDES%") == 0) { + while(_pack_file_gets(db->handle, pf, line, 512) && + strlen(_alpm_strtrim(line))) { + char *linedup; + STRDUP(linedup, _alpm_strtrim(line), goto error); + info->provides = alpm_list_add(info->provides, linedup); + } + } + } + _pack_close_file(pf); + pf = NULL; + } + + /* DELTAS */ + if(inforeq & INFRQ_DELTAS) { + if((pf = _pack_open_file(db->handle, pkgname, "deltas", PO_READ)) != NULL) { + while(!_pack_file_eof(db->handle, pf)) { + _pack_file_gets(db->handle, pf, line, 255); + _alpm_strtrim(line); + if(strcmp(line, "%DELTAS%") == 0) { + while(_pack_file_gets(db->handle, pf, line, 512) && + strlen(_alpm_strtrim(line))) { + pmdelta_t *delta = _alpm_delta_parse(line); + if(delta) { + info->deltas = alpm_list_add(info->deltas, delta); + } + } + } + } + _pack_close_file(pf); + pf = NULL; + } + } + /* INSTALL */ + if(inforeq & INFRQ_SCRIPTLET) { + if((pf = _pack_open_file(db->handle, pkgname, "depends", PO_READ)) != NULL) { + info->scriptlet = 1; + _pack_close_file(pf); + } + } + + /* internal */ + info->infolevel |= inforeq; + + return(0); + +error: + if(pf) { + _pack_close_file(pf); + } + return(-1); +} + +int _alpm_db_write(pmdb_t *db, pmpkg_t *info, pmdbinfrq_t inforeq) +{ + char pkgname[PATH_MAX]; + alpm_list_t *lp = NULL; + int retval = 0; + int local = 0; + pack_file_t *pf; + char buffer[1024]; + size_t len; + + ALPM_LOG_FUNC; + + if(db == NULL || info == NULL) { + return(-1); + } + + if(strcmp(db->treename, "local") == 0) { + local = 1; + } + + snprintf(pkgname, PATH_MAX, "%s-%s", info->name, info->version); + + /* DESC */ + if(inforeq & INFRQ_DESC) { + _alpm_log(PM_LOG_DEBUG, "writing %s-%s DESC information back to db\n", + info->name, info->version); + if((pf = _pack_open_file(db->handle, pkgname, "desc", PO_WRITE)) == NULL) { + _alpm_log(PM_LOG_ERROR, _("could not open package %s\n"), pkgname); + retval = -1; + goto cleanup; + } + + len = snprintf(buffer, 1024, "%%NAME%%\n%s\n\n" + "%%VERSION%%\n%s\n\n", info->name, info->version); + _pack_file_puts(db->handle, pf, buffer, len); + if(info->desc) { + len = snprintf(buffer, 1024, "%%DESC%%\n" + "%s\n\n", info->desc); + _pack_file_puts(db->handle, pf, buffer, len); + } + if(info->groups) { + _pack_file_puts(db->handle, pf, "%GROUPS%\n", 9); + for(lp = info->groups; lp; lp = lp->next) { + len = snprintf(buffer, 1024, "%s\n", (char *)lp->data); + _pack_file_puts(db->handle, pf, buffer, len); + } + _pack_file_puts(db->handle, pf, "\n", 1); + } + if(info->replaces) { + _pack_file_puts(db->handle, pf, "%REPLACES%\n", 11); + for(lp = info->replaces; lp; lp = lp->next) { + len = snprintf(buffer, 1024, "%s\n", (char *)lp->data); + _pack_file_puts(db->handle, pf, buffer, len); + } + len = snprintf(buffer, 1024, "\n"); + _pack_file_puts(db->handle, pf, buffer, len); + } + if(info->force) { + len = snprintf(buffer, 1024, "%%FORCE%%\n\n"); + _pack_file_puts(db->handle, pf, buffer, len); + } + if(local) { + if(info->url) { + len = snprintf(buffer, 1024, "%%URL%%\n" + "%s\n\n", info->url); + _pack_file_puts(db->handle, pf, buffer, len); + } + if(info->licenses) { + _pack_file_puts(db->handle, pf, "%LICENSE%\n", 10); + for(lp = info->licenses; lp; lp = lp->next) { + len = snprintf(buffer, 1024, "%s\n", (char *)lp->data); + _pack_file_puts(db->handle, pf, buffer, len); + } + len = snprintf(buffer, 1024, "\n"); + _pack_file_puts(db->handle, pf, buffer, len); + } + if(info->arch) { + len = snprintf(buffer, 1024, "%%ARCH%%\n" + "%s\n\n", info->arch); + _pack_file_puts(db->handle, pf, buffer, len); + } + if(info->builddate) { + len = snprintf(buffer, 1024, "%%BUILDDATE%%\n" + "%ju\n\n", (uintmax_t)info->builddate); + _pack_file_puts(db->handle, pf, buffer, len); + } + if(info->installdate) { + len = snprintf(buffer, 1024, "%%INSTALLDATE%%\n" + "%ju\n\n", (uintmax_t)info->installdate); + _pack_file_puts(db->handle, pf, buffer, len); + } + if(info->packager) { + len = snprintf(buffer, 1024, "%%PACKAGER%%\n" + "%s\n\n", info->packager); + _pack_file_puts(db->handle, pf, buffer, len); + } + if(info->isize) { + /* only write installed size, csize is irrelevant once installed */ + len = snprintf(buffer, 1024, "%%SIZE%%\n" + "%ju\n\n", (intmax_t)info->isize); + _pack_file_puts(db->handle, pf, buffer, len); + } + if(info->reason) { + len = snprintf(buffer, 1024, "%%REASON%%\n" + "%u\n\n", info->reason); + _pack_file_puts(db->handle, pf, buffer, len); + } + } else { + if(info->size) { + len = snprintf(buffer, 1024, "%%CSIZE%%\n" + "%ju\n\n", (intmax_t)info->size); + _pack_file_puts(db->handle, pf, buffer, len); + } + if(info->isize) { + len = snprintf(buffer, 1024, "%%ISIZE%%\n" + "%ju\n\n", (intmax_t)info->isize); + _pack_file_puts(db->handle, pf, buffer, len); + } + if(info->md5sum) { + len = snprintf(buffer, 1024, "%%MD5SUM%%\n" + "%s\n\n", info->md5sum); + _pack_file_puts(db->handle, pf, buffer, len); + } + } + _pack_close_file(pf); + pf = NULL; + } + + /* FILES */ + if(local && (inforeq & INFRQ_FILES)) { + _alpm_log(PM_LOG_DEBUG, "writing %s-%s FILES information back to db\n", + info->name, info->version); + if((pf = _pack_open_file(db->handle, pkgname, "files", PO_WRITE)) == NULL) { + _alpm_log(PM_LOG_ERROR, _("could not open package %s\n"), pkgname); + retval = -1; + goto cleanup; + } + + if(info->files) { + len = snprintf(buffer, 1024, "%%FILES%%\n"); + _pack_file_puts(db->handle, pf, buffer, len); + for(lp = info->files; lp; lp = lp->next) { + len = snprintf(buffer, 1024, "%s\n", (char *)lp->data); + _pack_file_puts(db->handle, pf, buffer, len); + } + len = snprintf(buffer, 1024, "\n"); + _pack_file_puts(db->handle, pf, buffer, len); + } + if(info->backup) { + len = snprintf(buffer, 1024, "%%BACKUP%%\n"); + _pack_file_puts(db->handle, pf, buffer, len); + for(lp = info->backup; lp; lp = lp->next) { + len = snprintf(buffer, 1024, "%s\n", (char *)lp->data); + _pack_file_puts(db->handle, pf, buffer, len); + } + len = snprintf(buffer, 1024, "\n"); + _pack_file_puts(db->handle, pf, buffer, len); + } + _pack_close_file(pf); + pf = NULL; + } + + /* DEPENDS */ + if(inforeq & INFRQ_DEPENDS) { + _alpm_log(PM_LOG_DEBUG, "writing %s-%s DEPENDS information back to db\n", + info->name, info->version); + if((pf = _pack_open_file(db->handle, pkgname, "depends", PO_WRITE)) == NULL) { + _alpm_log(PM_LOG_ERROR, _("could not open package %s\n"), pkgname); + retval = -1; + goto cleanup; + } + + if(info->depends) { + _pack_file_puts(db->handle, pf, "%DEPENDS%\n", 10); + for(lp = info->depends; lp; lp = lp->next) { + char *depstring = alpm_dep_get_string(lp->data); + len = snprintf(buffer, 1024, "%s\n", depstring); + _pack_file_puts(db->handle, pf, buffer, len); + free(depstring); + } + len = snprintf(buffer, 1024, "\n"); + _pack_file_puts(db->handle, pf, buffer, len); + } + if(info->optdepends) { + _pack_file_puts(db->handle, pf, "%OPTDEPENDS%\n", 13); + for(lp = info->optdepends; lp; lp = lp->next) { + len = snprintf(buffer, 1024, "%s\n", (char *)lp->data); + _pack_file_puts(db->handle, pf, buffer, len); + } + len = snprintf(buffer, 1024, "\n"); + _pack_file_puts(db->handle, pf, buffer, len); + } + if(info->conflicts) { + _pack_file_puts(db->handle, pf, "%CONFLICTS%\n", 12); + for(lp = info->conflicts; lp; lp = lp->next) { + len = snprintf(buffer, 1024, "%s\n", (char *)lp->data); + _pack_file_puts(db->handle, pf, buffer, len); + } + len = snprintf(buffer, 1024, "\n"); + _pack_file_puts(db->handle, pf, buffer, len); + } + if(info->provides) { + _pack_file_puts(db->handle, pf, "%PROVIDES%\n", 11); + for(lp = info->provides; lp; lp = lp->next) { + len = snprintf(buffer, 1024, "%s\n", (char *)lp->data); + _pack_file_puts(db->handle, pf, buffer, len); + } + len = snprintf(buffer, 1024, "\n"); + _pack_file_puts(db->handle, pf, buffer, len); + } + _pack_close_file(pf); + pf = NULL; + } + + /* INSTALL */ + /* nothing needed here (script is automatically extracted) */ + +cleanup: + if(pf) { + _pack_close_file(pf); + } + + return(retval); +} + +int _alpm_db_remove(pmdb_t *db, pmpkg_t *info) +{ + int ret = 0; + char pkgname[512]; + + ALPM_LOG_FUNC; + + snprintf(pkgname, 512, "%s-%s", info->name, info->version); + _pack_package_remove(db->handle, pkgname); + + if(db == NULL || info == NULL) { + RET_ERR(PM_ERR_DB_NULL, -1); + } + + return(ret); +} + +void *_alpm_db_changelog_open(pmdb_t *db, pmpkg_t *pkg) +{ + char name[512]; + snprintf(name, 512, "%s-%s", pkg->name, pkg->version); + return _pack_open_file(db->handle, name, "changelog", PO_READ); +} + +int _alpm_db_changelog_close(pmdb_t *db, void *stream) +{ + _pack_close_file(stream); + return 0; +} + +size_t _alpm_db_changelog_read(pmdb_t *db, const void *pf, void *ptr, size_t size) +{ + return _pack_file_gets(db->handle, pf, ptr, size); +} + +/* vim: set ts=2 sw=2 noet: */ diff --git a/lib/libalpm/be_packed.h b/lib/libalpm/be_packed.h new file mode 100644 index 0000000..e118a3a --- /dev/null +++ b/lib/libalpm/be_packed.h @@ -0,0 +1,757 @@ +/* + * Copyright (c) 2008 Sivert Berg + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/* _HOW IT WORKS_ + * + * If you've ever looked at the ext2 filesystem you will find alot + * of similarities (among others the inode that I didn't have the + * imagination to find a new name for). + * + * _GROUP_ + * Basicly the database is divided into groups. Each group got the + * same number of blocks and inodes. + * + * _INODE_ + * Each file is assigned an unique inode. The inode got a list of + * blocks where the actual data for the file is stored. Worth noting + * is that the last entry in the block list is indirect. That means + * it points to a block that points to blocks. + */ + +#include <stdint.h> +#include <sys/mman.h> +#include <stdlib.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <stdio.h> +#include <string.h> +#include <unistd.h> + + +#ifndef BE_PACKED_H +#define BE_PACKED_H + +#define BLOCK_SIZE 256 +#define GROUP_SIZE (BLOCK_SIZE * 4096) +#define BLOCKS_PER_GROUP 3936 +#define INODES_PER_GROUP 1024 + +#define MIN(a, b) (((a) < (b))?(a):(b)) + +/* _pack_open_file flags */ +#define PO_READ 1 +#define PO_WRITE 2 + + +typedef struct __inode_t { + int32_t size; + int32_t block[7]; +} inode_t; + +typedef struct __group_desc_t { + uint32_t inodes_free; + uint32_t blocks_free; + uint32_t checksum; + inode_t i_table[INODES_PER_GROUP]; + uint32_t bitmap[BLOCKS_PER_GROUP / 32]; + char data[]; +} group_desc_t; + +typedef struct __package_t { + int32_t inode; + uint16_t rec_len; + uint16_t name_len; + char name[]; +} package_t; + +typedef struct __pack_db_t { + int fd; + int groups; + int read_only; + group_desc_t *map; +} pack_db_t; + +typedef struct __pack_file_t { + int inode; + size_t pos; +} pack_file_t; + +typedef struct __pack_dir_t { + size_t pos; +} pack_dir_t; + + +/** + ** GROUP FUNCTIONS + */ +/* Initilizes a new group */ +void _pack_init_group(group_desc_t *group) +{ + int i, j; + + for (i = 0; i < INODES_PER_GROUP; i++) { + group->i_table[i].size = -1; + for (j = 0; j < 7; j++) + group->i_table[i].block[j] = -1; + } + + for (i = 0; i < BLOCKS_PER_GROUP / 32; i++) { + group->bitmap[i] = 0; + } + + group->inodes_free = INODES_PER_GROUP; + group->blocks_free = BLOCKS_PER_GROUP; + group->checksum = 0; +} + + +/* Adds a new group to the file */ +void _pack_new_group(pack_db_t *db) +{ + db->groups++; + ftruncate(db->fd, db->groups * GROUP_SIZE); + db->map = mremap(db->map, (db->groups - 1) * GROUP_SIZE, + db->groups * GROUP_SIZE, MREMAP_MAYMOVE); + + _pack_init_group((group_desc_t*)((char*)db->map + + GROUP_SIZE * (db->groups - 1))); +} + + +/** + ** POINTER FUNCTIONS (return pointers to various structures) + */ +group_desc_t *_pack_get_group(pack_db_t *db, int group) +{ + return (group_desc_t*)((char*)db->map + group*GROUP_SIZE); +} + + +inode_t *_pack_get_inode(pack_db_t *db, int inode) +{ + int group = inode / INODES_PER_GROUP; + group_desc_t *groupd = _pack_get_group(db, group); + inode %= INODES_PER_GROUP; + + return groupd->i_table + inode; +} + +void *_pack_get_block(pack_db_t *db, int block) +{ + if (block == -1) + return NULL; + + group_desc_t *group = _pack_get_group(db, block / BLOCKS_PER_GROUP); + return group->data + (block % BLOCKS_PER_GROUP) * BLOCK_SIZE; +} + + +/* Searches for a free block in the bitmap and marks the bit + * it finds as used. + */ +int _pack_check_bitmap(group_desc_t *group) +{ + int i = 0, j = 0, t = 0; + + for (i = 0; i < BLOCKS_PER_GROUP / 32; i++) { + if (group->bitmap[i] != 0xffffffff) { + t = group->bitmap[i]; + for (j = 0; j < 32 && (t & 1); j++, t >>= 1); + break; + } + } + + group->bitmap[i] |= 1 << j; + return i * 32 + j; +} + + +/* Clears the n'th bit */ +int _pack_bitmap_free(group_desc_t *group, int n) +{ + int i = n / 32; + int j = n % 32; + + group->bitmap[i] &= ~(i << j); + return 0; +} + + +/* Returns the number of a new unused block + * CAUTION: THIS FUNCTION MAY RESULT IN A MMREMAP! + * that means ANY pointers into the database may be invalid + * after you've called this function. + */ +int _pack_get_free_block(pack_db_t *db) +{ + int i; + group_desc_t *group; + int ret; + + for (i = 0; i < db->groups; i++) { + group = _pack_get_group(db, i); + if (group->blocks_free) { + group->blocks_free--; + ret = _pack_check_bitmap(group); + ret += i * BLOCKS_PER_GROUP; + return ret;; + } + } + + _pack_new_group(db); + group = _pack_get_group(db, i); + group->blocks_free--; + ret = _pack_check_bitmap(group); + ret += i * BLOCKS_PER_GROUP; + + return ret; +} + + +/* Frees the block given */ +int _pack_block_free(pack_db_t *db, int block) +{ + group_desc_t *group; + int g, b; + + g = block / BLOCKS_PER_GROUP; + b = block & BLOCKS_PER_GROUP; + + group = _pack_get_group(db, g); + group->blocks_free++; + + return _pack_bitmap_free(group, b); +} + + +/* Returns the number of a new free inode. + * CAUTION: THIS FUNCTION MAY RESULT IN A MMREMAP! + * that means ANY pointers into the database may be invalid + * after you've called this function. + */ +int _pack_get_free_inode(pack_db_t *db) +{ + int i = 0, j = 0; + group_desc_t *group; + + for (i = 0; i < db->groups; i++) { + group = _pack_get_group(db, i); + if (group->inodes_free) { + group->inodes_free--; + for (j = 0; j < INODES_PER_GROUP; j++) { + if (group->i_table[j].size == -1) { + group->i_table[j].size = 0; + return i * INODES_PER_GROUP + j; + } + } + } + } + + _pack_new_group(db); + group = _pack_get_group(db, i); + group->inodes_free--; + + return i * INODES_PER_GROUP; +} + + +/* Frees an inode and all of the blocks it might be using */ +int _pack_inode_free(pack_db_t *db, int in) +{ + inode_t *inode = _pack_get_inode(db, in); + group_desc_t *group = _pack_get_group(db, in / INODES_PER_GROUP); + int i; + int block; + int32_t *blocks; + int size = BLOCK_SIZE / sizeof(uint32_t) - 1; + + inode->size = -1; + + for (i = 0; i < 6; i++) { + if (inode->block[i] != -1) { + _pack_block_free(db, inode->block[i]); + inode->block[i] = -1; + } + } + + block = inode->block[6]; + blocks = _pack_get_block(db, block); + + while(blocks != NULL) { + for (i = 0; i < size; i++) { + if (blocks[i] != -1) + _pack_block_free(db, blocks[i]); + } + + _pack_block_free(db, block); + block = blocks[size]; + blocks = _pack_get_block(db, block); + } + + group->inodes_free++; + + return 0; +} + + +/* Returns the number of the block associated with the inode */ +int _pack_inode_get_block(pack_db_t *db, int i, int block) +{ + inode_t *inode = _pack_get_inode(db, i); + if (block < 6) + return inode->block[block]; + + block -= 6; + int size = BLOCK_SIZE / sizeof(uint32_t) - 1; + uint32_t *blocks = _pack_get_block(db, inode->block[6]); + + for (;blocks != NULL && block >= size; block -= size) + blocks = _pack_get_block(db, blocks[size]); + + if (blocks == NULL) + return -1; + + return blocks[block]; +} + + +/* Returns the number of the block block associated with the inode. + * If there is no such block, it gets allocated. + * CAUTION: THIS FUNCTION MAY RESULT IN A MMREMAP! + * that means ANY pointers into the database may be invalid + * after you've called this function. + */ +int _pack_inode_add_block(pack_db_t *db, int in, int block) +{ + inode_t *inode = _pack_get_inode(db, in); + int tmp; + + if (block < 6) { + if (inode->block[block] != -1) + return inode->block[block]; + + tmp = _pack_get_free_block(db); + inode = _pack_get_inode(db, in); + return inode->block[block] = tmp; + } + + block -= 6; + int size = BLOCK_SIZE / sizeof(int32_t) - 1; + int i; + int old_block; + int32_t *blocks; + + if (inode->block[6] == -1) { + tmp = _pack_get_free_block(db); + inode = _pack_get_inode(db, in); + inode->block[6] = tmp; + blocks = _pack_get_block(db, inode->block[6]); + for (i = 0; i < size + 1; i++) + blocks[i] = -1; + } + else + blocks = _pack_get_block(db, inode->block[6]); + + old_block = inode->block[6]; + + for (;block >= size; block -= size) { + if (blocks[size] == -1) { + tmp = _pack_get_free_block(db); + blocks = _pack_get_block(db, old_block); + blocks[size] = tmp; + old_block = tmp; + blocks = _pack_get_block(db, tmp); + for (i = 0; i < size + 1; i++) + blocks[i] = -1; + } + else { + old_block = blocks[size]; + blocks = _pack_get_block(db, blocks[size]); + } + } + + if (blocks[block] == -1) { + tmp = _pack_get_free_block(db); + blocks = _pack_get_block(db, old_block); + blocks[block] = tmp; + } + + return blocks[block]; +} + +/* Returns a pointer to the package with the name pkgname. + * Returns NULL if it's not found + */ +package_t *_pack_find_package(pack_db_t *db, const char *pkgname) +{ + inode_t *inode = _pack_get_inode(db, 0); + int s_len = strlen(pkgname); + package_t *pkg; + int block = 0; + int b_pos = 0; + + pkg = _pack_get_block(db, _pack_inode_get_block(db, 0, block)); + + while (block < (inode->size / BLOCK_SIZE)) { + if (pkg->inode != 0 && + pkg->name_len == s_len && + strncmp(pkgname, pkg->name, s_len) == 0) + break; + + b_pos += pkg->rec_len; + pkg = (package_t*)((char*)pkg + pkg->rec_len); + if (b_pos >= BLOCK_SIZE) { + pkg = _pack_get_block(db, _pack_inode_get_block(db, 0, ++block)); + b_pos = 0; + } + } + + return pkg; +} + +/* Creates a new package with the name pkgname. + * Returns a pointer to the new package. + * CAUTION: THIS FUNCTION MAY RESULT IN A MMREMAP! + * that means ANY pointers into the database may be invalid + * after you've called this function. + */ +package_t *_pack_add_package(pack_db_t *db, const char *pkgname) +{ + inode_t *inode = _pack_get_inode(db, 0); + int s_len = strlen(pkgname); + int len = s_len + sizeof(package_t); + package_t *pkg; + int block = 0; + int b_pos = 0; + + pkg = _pack_get_block(db, _pack_inode_get_block(db, 0, block)); + + while (pkg && block < (inode->size / BLOCK_SIZE)) { + if (pkg->rec_len >= len && pkg->inode == 0) + break; + + b_pos += pkg->rec_len; + pkg = (package_t*)((char*)pkg + pkg->rec_len); + if (b_pos >= BLOCK_SIZE) { + pkg = _pack_get_block(db, _pack_inode_get_block(db, 0, ++block)); + b_pos = 0; + } + } + + if (pkg == NULL) { + pkg = _pack_get_block(db, _pack_inode_add_block(db, 0, block)); + pkg->rec_len = BLOCK_SIZE; + pkg->inode = 0; + inode = _pack_get_inode(db, 0); + inode->size += BLOCK_SIZE; + } + + strncpy(pkg->name, pkgname, s_len); + if (pkg->rec_len > (len + sizeof(package_t) + 10)) { + package_t *foo = (package_t*)((char*)pkg + len); + foo->rec_len = pkg->rec_len - len; + foo->inode = 0; + pkg->rec_len = len; + } + + pkg->inode = -1; + pkg->name_len = s_len; + + return pkg; +} + + +/* Reads data from the file until buf is full or we hit a newline */ +int _pack_file_gets(pack_db_t *db, pack_file_t *file, char *buf, size_t size) +{ + inode_t *inode = _pack_get_inode(db, file->inode); + size_t i; + size_t b_pos = file->pos % BLOCK_SIZE; + const char *src = _pack_get_block(db, + _pack_inode_get_block(db, file->inode, file->pos / BLOCK_SIZE)); + int newline = 0; + + for (i = 0 + ;i < size && file->pos < inode->size && !newline + ;i++, b_pos++, file->pos++) { + if (b_pos >= BLOCK_SIZE) { + src = _pack_get_block(db, + _pack_inode_get_block(db, file->inode, file->pos / BLOCK_SIZE)); + b_pos = 0; + } + + buf[i] = src[b_pos]; + + if (src[b_pos] == '\n') + newline = 1; + } + + if (i < size - 1) + buf[i] = '\0'; + + return i; +} + + +/* Write the data in buf into the file + * CAUTION: THIS FUNCTION MAY RESULT IN A MMREMAP! + * that means ANY pointers into the database may be invalid + * after you've called this function. + */ +int _pack_file_puts(pack_db_t *db, pack_file_t *file, + const char *buf, size_t size) +{ + size_t i; + size_t b_pos = file->pos % BLOCK_SIZE; + char *dest = _pack_get_block(db, + _pack_inode_add_block(db, file->inode, file->pos / BLOCK_SIZE)); + int block; + + for (i = 0; i < size; i++, file->pos++, b_pos++) + { + if (b_pos >= BLOCK_SIZE) { + block = _pack_inode_add_block(db, file->inode, file->pos / BLOCK_SIZE); + dest = _pack_get_block(db, block); + b_pos = 0; + } + dest[b_pos] = buf[i]; + } + + inode_t *inode = _pack_get_inode(db, file->inode); + inode->size = file->pos; + + return i; +} + + +int _pack_file_eof(pack_db_t *db, pack_file_t *file) +{ + inode_t *inode = _pack_get_inode(db, file->inode); + return file->pos >= inode->size; +} + + +/* Opens a file in the database + * flags: if you pass PO_READ, it will return -1 + * if the file does not exist, if you pass PO_WRITE + * the file will be created if it does not exist + * CAUTION: THIS FUNCTION MAY RESULT IN A MMREMAP! + * that means ANY pointers into the database may be invalid + * after you've called this function. + */ +pack_file_t *_pack_open_file(pack_db_t *db, const char *pkgname, + const char *filename, int flags) +{ + char name[512]; + + snprintf(name, 512, "%s/%s", pkgname, filename); + + package_t *pkg = _pack_find_package(db, name); + int w = flags & PO_WRITE; + int tmp; + + if (w && db->read_only) + return NULL; + + if (pkg == NULL && w) + pkg = _pack_add_package(db, name); + else if (pkg == NULL) + return NULL; + + pack_file_t *ret = malloc(sizeof(pack_file_t)); + ret->pos = 0; + + if (pkg->inode == -1 && w) { + tmp = _pack_get_free_inode(db); + pkg = _pack_find_package(db, name); + ret->inode = pkg->inode = tmp; + } + else + ret->inode = pkg->inode; + + return ret; +} + + +/* Closes the file given */ +void _pack_close_file(pack_file_t *file) +{ + free(file); +} + + +/* Removes a package along with all it's files */ +int _pack_package_remove(pack_db_t *db, const char *pkgname) +{ + inode_t *inode = _pack_get_inode(db, 0); + int s_len = strlen(pkgname); + package_t *prev, *pkg; + int block = 0; + int b_pos = 0; + + if (db->read_only) + return -1; + + prev = NULL; + pkg = _pack_get_block(db, _pack_inode_get_block(db, 0, block)); + + while (block < (inode->size / BLOCK_SIZE)) { + if (pkg->inode != 0 && + pkg->name_len > s_len && + strncmp(pkgname, pkg->name, s_len) == 0 && + pkg->name[s_len] == '/') { + _pack_inode_free(db, pkg->inode); + + if (prev != NULL && prev->inode == 0) { + prev->rec_len += pkg->rec_len; + pkg = prev; + } + else + pkg->inode = 0; + } + + prev = pkg; + b_pos += pkg->rec_len; + pkg = (package_t*)((char*)pkg + pkg->rec_len); + if (b_pos >= BLOCK_SIZE) { + pkg = _pack_get_block(db, _pack_inode_get_block(db, 0, ++block)); + b_pos = 0; + prev = NULL; + } + } + + return 0; +} + + +/* Gets the next entry in the directory */ +int _pack_dir_next(pack_db_t *db, pack_dir_t *dir, char *name, size_t len) +{ + inode_t *inode = _pack_get_inode(db, 0); + + if (dir->pos >= inode->size) + return -1; + + int block, b_pos; + package_t *pkg; + + do { + block = dir->pos / BLOCK_SIZE; + b_pos = dir->pos % BLOCK_SIZE; + pkg = _pack_get_block(db, _pack_inode_get_block(db, 0, block)); + if (!pkg) + return -1; + pkg = (package_t*)((char*)pkg + b_pos); + dir->pos += pkg->rec_len; + } while (pkg->inode == 0); + + strncpy(name, pkg->name, MIN(len, pkg->name_len)); + if (len > pkg->name_len) + name[pkg->name_len] = '\0'; + + return 0; +} + + +/* Opens the directory for reading */ +pack_dir_t *_pack_open_dir(pack_db_t *db) +{ + pack_dir_t *dir = malloc(sizeof(pack_dir_t)); + dir->pos = 0; + + return dir; +} + + +/* Closes the given directory */ +void _pack_close_dir(pack_dir_t *dir) +{ + free(dir); +} + + +/* Opens a packed database */ +pack_db_t *_pack_db_open(const char *filename) +{ + struct stat st; + int is_new = 0; + + pack_db_t *db = malloc(sizeof(pack_db_t)); + db->fd = open(filename, O_RDWR, 0); + db->read_only = 0; + + if (db->fd == -1) { + db->fd = open(filename, O_RDONLY, 0); + if (db->fd == -1) { + db->fd = open(filename, O_RDWR | O_CREAT, 0644); + + if (db->fd == -1) { + free(db); + printf("Could not open %s\n", filename); + return NULL; + } + + ftruncate(db->fd, GROUP_SIZE); + is_new = 1; + } + else + db->read_only = 1; + } + + fstat(db->fd, &st); + db->groups = st.st_size / GROUP_SIZE; + + int flags = PROT_READ; + if (!db->read_only) + flags |= PROT_WRITE; + + db->map = mmap(NULL, GROUP_SIZE * db->groups, flags, MAP_SHARED, + db->fd, 0); + + if (db->map == MAP_FAILED) { + perror("mmap"); + free(db); + exit(0); + } + + if (is_new) { + _pack_init_group(db->map); + db->map->i_table[0].size = 0; + } + + return db; +} + + +/* Closes a packed database */ +void _pack_db_close(pack_db_t *db) +{ + munmap(db->map, db->groups * GROUP_SIZE); + close(db->fd); + free(db); +} + +#endif /* BE_PACKED_H */ diff --git a/lib/libalpm/db.h b/lib/libalpm/db.h index 96fac0d..96291fc 100644 --- a/lib/libalpm/db.h +++ b/lib/libalpm/db.h @@ -62,6 +62,9 @@ int _alpm_db_populate(pmdb_t *db); int _alpm_db_read(pmdb_t *db, pmpkg_t *info, pmdbinfrq_t inforeq); int _alpm_db_write(pmdb_t *db, pmpkg_t *info, pmdbinfrq_t inforeq); int _alpm_db_remove(pmdb_t *db, pmpkg_t *info); +void *_alpm_db_changelog_open(pmdb_t *db, pmpkg_t *pkg); +int _alpm_db_changelog_close(pmdb_t *db, void *stream); +size_t _alpm_db_changelog_read(pmdb_t *db, const void *pf, void *ptr, size_t size); #endif /* _ALPM_DB_H */ diff --git a/lib/libalpm/package.c b/lib/libalpm/package.c index ee0ff6f..7ab9710 100644 --- a/lib/libalpm/package.c +++ b/lib/libalpm/package.c @@ -447,13 +447,7 @@ void SYMEXPORT *alpm_pkg_changelog_open(pmpkg_t *pkg) ASSERT(pkg != NULL, return(NULL)); if(pkg->origin == PKG_FROM_CACHE) { - char clfile[PATH_MAX]; - snprintf(clfile, PATH_MAX, "%s/%s/%s-%s/changelog", - alpm_option_get_dbpath(), - alpm_db_get_name(handle->db_local), - alpm_pkg_get_name(pkg), - alpm_pkg_get_version(pkg)); - return fopen(clfile, "r"); + return(_alpm_db_changelog_open(handle->db_local, pkg)); } else if(pkg->origin == PKG_FROM_FILE) { struct archive *archive = NULL; struct archive_entry *entry; @@ -500,7 +494,7 @@ size_t SYMEXPORT alpm_pkg_changelog_read(void *ptr, size_t size, { size_t ret = 0; if(pkg->origin == PKG_FROM_CACHE) { - ret = fread(ptr, 1, size, (FILE*)fp); + ret = _alpm_db_changelog_read(handle->db_local, fp, ptr, size); } else if(pkg->origin == PKG_FROM_FILE) { ret = archive_read_data((struct archive*)fp, ptr, size); } @@ -533,7 +527,7 @@ int SYMEXPORT alpm_pkg_changelog_close(const pmpkg_t *pkg, void *fp) { int ret = 0; if(pkg->origin == PKG_FROM_CACHE) { - ret = fclose((FILE*)fp); + ret = _alpm_db_changelog_close(handle->db_local, fp); } else if(pkg->origin == PKG_FROM_FILE) { ret = archive_read_finish((struct archive *)fp); } diff --git a/src/util/Makefile.am b/src/util/Makefile.am index 97a0ffa..64e18ad 100644 --- a/src/util/Makefile.am +++ b/src/util/Makefile.am @@ -3,7 +3,7 @@ conffile = ${sysconfdir}/pacman.conf dbpath = ${localstatedir}/lib/pacman/ cachedir = ${localstatedir}/cache/pacman/pkg/ -bin_PROGRAMS = vercmp testpkg testdb +bin_PROGRAMS = vercmp testpkg testdb convdb DEFS = -DLOCALEDIR=\"@localedir@\" \ -DCONFFILE=\"$(conffile)\" \ @@ -18,6 +18,8 @@ AM_CFLAGS = -pedantic -D_GNU_SOURCE vercmp_SOURCES = vercmp.c vercmp_LDADD = $(top_builddir)/lib/libalpm/.libs/libalpm.la +convdb_SOURCES = convdb.c + testpkg_SOURCES = testpkg.c testpkg_LDADD = $(top_builddir)/lib/libalpm/.libs/libalpm.la diff --git a/src/util/convdb.c b/src/util/convdb.c new file mode 100644 index 0000000..f1f39bd --- /dev/null +++ b/src/util/convdb.c @@ -0,0 +1,236 @@ +/* + * Copyright (c) 2008 Sivert Berg + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + + +/* WARNING: This file is pretty much just a quick hack thrown together + * in a hurry. It seems to work, but don't take my word for it. + */ + +#include <be_packed.h> +#include <dirent.h> +#include <archive.h> +#include <archive_entry.h> + + +/* Convert an archieved database into a packed database + * Expects a gziped database ending in .db.tar.gz + */ +void convert_archive(const char *filename) +{ + struct archive_entry *aent = NULL; + struct archive *arch = archive_read_new(); + archive_read_support_compression_gzip(arch); + archive_read_support_format_tar(arch); + if (archive_read_open_filename(arch, filename, 1) != ARCHIVE_OK) { + printf("Could not open archive %s\n", archive_error_string(arch)); + return; + } + + char *db_name = strndup(filename, strlen(filename) - strlen(".tar.gz")); + pack_db_t *db = _pack_db_open(db_name); + free(db_name); + char buffer[4096]; + size_t size; + pack_file_t *pf; + + while (!archive_read_next_header(arch, &aent)) { + const char *type = NULL; + if (strstr(archive_entry_pathname(aent), "/desc")) + type = "desc"; + else if (strstr(archive_entry_pathname(aent), "/depends")) + type = "depends"; + else if (strstr(archive_entry_pathname(aent), "/install")) + type = "install"; + else if (strstr(archive_entry_pathname(aent), "/files")) + type = "files"; + + if (type == NULL) + continue; + + char *name = strdup(archive_entry_pathname(aent)); + int i; + for (i = 0; name[i] != '/' && name[i]; i++); + name[i] = '\0'; + + pf = _pack_open_file(db, name, type, PO_WRITE); + + free(name); + + do { + size = archive_read_data(arch, buffer, 4096); + _pack_file_puts(db, pf, buffer, size); + } while (size == 4096); + + _pack_close_file(pf); + + } + + _pack_db_close(db); + archive_read_close(arch); + archive_read_finish(arch); +} + + + +const char *dir_name(const char *full_path) +{ + const char *slash = full_path; + while (*full_path) + if (*full_path++ == '/') + slash = full_path; + return slash; +} + + +/* Convert a directory database into a packed database */ +void convert_directory(const char *pathname) +{ + char db_name[1024]; + pack_db_t *db; + DIR *dir = opendir(pathname); + struct dirent *dire; + char path[1024]; + char buffer[8196]; + size_t size; + FILE *f; + pack_file_t *pf; + + strcpy(db_name, dir_name(pathname)); + strcat(db_name, ".db"); + db = _pack_db_open(db_name); + + if (dir == NULL) { + perror("opendir"); + return; + } + + dire = readdir(dir); + while (dire != NULL) { + if (strcmp(dire->d_name, ".") && strcmp(dire->d_name, "..")) { + + + size_t foo = 0; + sprintf(path, "%s/%s/desc", pathname, dire->d_name); + f = fopen(path, "r"); + if (f == NULL) + goto depends; + pf = _pack_open_file(db, dire->d_name, "desc", PO_WRITE); + do { + size = fread(buffer, 1, 8196, f); + _pack_file_puts(db, pf, buffer, size); + foo += size; + } while (size == 8196); + _pack_close_file(pf); + fclose(f); +depends: + sprintf(path, "%s/%s/depends", pathname, dire->d_name); + f = fopen(path, "r"); + if (f == NULL) + goto files; + pf = _pack_open_file(db, dire->d_name, "depends", PO_WRITE); + do { + size = fread(buffer, 1, 8196, f); + _pack_file_puts(db, pf, buffer, size); + foo += size; + } while (size == 8196); + _pack_close_file(pf); + fclose(f); +files: + sprintf(path, "%s/%s/files", pathname, dire->d_name); + f = fopen(path, "r"); + if (f == NULL) + goto install; + pf = _pack_open_file(db, dire->d_name, "files", PO_WRITE); + do { + size = fread(buffer, 1, 8196, f); + _pack_file_puts(db, pf, buffer, size); + foo += size; + } while (size == 8196); + _pack_close_file(pf); + fclose(f); +install: + sprintf(path, "%s/%s/install", pathname, dire->d_name); + f = fopen(path, "r"); + if (f == NULL) + goto changelog; + pf = _pack_open_file(db, dire->d_name, "install", PO_WRITE); + do { + size = fread(buffer, 1, 8196, f); + _pack_file_puts(db, pf, buffer, size); + foo += size; + } while (size == 8196); + _pack_close_file(pf); + fclose(f); +changelog: + sprintf(path, "%s/%s/changelog", pathname, dire->d_name); + f = fopen(path, "r"); + if (f == NULL) + goto deltas; + pf = _pack_open_file(db, dire->d_name, "changelog", PO_WRITE); + do { + size = fread(buffer, 1, 8196, f); + _pack_file_puts(db, pf, buffer, size); + foo += size; + } while (size == 8196); + _pack_close_file(pf); + fclose(f); +deltas: + sprintf(path, "%s/%s/deltas", pathname, dire->d_name); + f = fopen(path, "r"); + if (f == NULL) + goto done; + pf = _pack_open_file(db, dire->d_name, "deltas", PO_WRITE); + do { + size = fread(buffer, 1, 8196, f); + _pack_file_puts(db, pf, buffer, size); + foo += size; + } while (size == 8196); + _pack_close_file(pf); + fclose(f); +done: + pf = NULL; + } + + dire = readdir(dir); + } + + closedir(dir); + _pack_db_close(db); +} + +int main(int argc, char *argv[]) +{ + if (argc < 2) { + printf("Usage: %s <dir/file>\n", argv[0]); + exit(0); + } + + if (strstr(argv[1], ".tar.gz") != NULL) + convert_archive(argv[1]); + else + convert_directory(argv[1]); + + return 0; +} -- 1.6.0.5

3 4

[pacman-dev] [PATCH] makepkg: ensure PKGBUILD does not contain CRLF characters
by Dan McGee 16 Dec '08

16 Dec '08

Do a simple check before sourcing the file to ensure we are a valid bash script. Signed-off-by: Dan McGee <dan(a)archlinux.org> --- scripts/makepkg.sh.in | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diff --git a/scripts/makepkg.sh.in b/scripts/makepkg.sh.in index b889625..179746d 100644 --- a/scripts/makepkg.sh.in +++ b/scripts/makepkg.sh.in @@ -1354,6 +1354,12 @@ if [ ! -f "$BUILDSCRIPT" ]; then # PKGBUILD passed through a pipe BUILDSCRIPT=/dev/stdin fi +else + crlftest=$(file $BUILDSCRIPT | grep -F 'CRLF' || true) + if [ "$crlftest" != "" ]; then + error "$(gettext "%s contains CRLF characters and cannot be sourced.")" "$BUILDSCRIPT" + exit 1 + fi fi source "$BUILDSCRIPT" -- 1.6.0.4

4 3

[pacman-dev] GPG work
by Dan McGee 15 Dec '08

15 Dec '08

I did quite a bit more work with GPG today. I wrapped my head around GPGME, which presents a nice C interface to the GPG stuff so we are now a lot closer to a working implementation: http://code.toofishes.net/gitweb.cgi?p=pacman.git;a=shortlog;h=refs/heads/n… >From the script side of things, I didn't change much. The libalpm code has changed considerably, and there is still a lot of room for improvement. Let me know if you guys have questions. -Dan

8 13

Re: [pacman-dev] makepkg: package splitting
by Marc - A. Dahlhaus [ Administration | Westermann GmbH ] 15 Dec '08

15 Dec '08

Am Montag, den 15.12.2008, 22:24 +1000 schrieb Allan McRae: > Marc - A. Dahlhaus [ Administration | Westermann GmbH ] wrote: > > Am Montag, den 15.12.2008, 21:43 +1000 schrieb Allan McRae: > > > > Nice, i'll test and report back... > > Is there any schedule on when it will be merged to mainline? > > > > > > The current schedule is when I tell Dan it ready... I want to add > documentation and clean up some of the output with the package splitting > but otherwise I am happy with it now. So, the answer is "soonish". > > Allan If you need any help, be it implementation of missing functionality or testing of something, let me know. I'll also try to come up with a option to build only a subset of the split-packages (if it makes sense) but haven't looked deeper into it yet. Marc

1 0

Re: [pacman-dev] makepkg: package splitting
by Marc - A. Dahlhaus [ Administration | Westermann GmbH ] 15 Dec '08

15 Dec '08

Am Dienstag, den 09.12.2008, 22:32 +1000 schrieb Allan McRae: > Marc - A. Dahlhaus [ Administration | Westermann GmbH ] wrote: > > > > I'll tested that again and i forgot to change two of the DESTDIR > > locations to the new home of the pkgdirs... So the missing cleanup was > > actually a stupid user error... > > > > You should only be doing something like "make DESTDIR=$pkgdir install". > Makepkg points $pkgdir in the right place for each split package. (I > will get around to documenting all this at some stage) > > Allan Hello Allan, i think i found another bug. Every package that is a split-build and not in the first position in the pkgname build-order shows an empty "Optional dependencies" string during installation. I think there is something wrong in restore_package_variables during variable resetting to initial values or in backup_package_variables during the backup declaration. It looks like there are to much quotes around the declaration so that it declares an empty string. Maybe a "-z" check before the actual variable declaration could fix that particular problem. Marc

2 2

[pacman-dev] Order of option parsing and sourcing makepkg.conf
by Allan McRae 14 Dec '08

14 Dec '08

Hi, With the commit which enables us to specify the makepkg config file on the command-line (http://projects.archlinux.org/?p=pacman.git;a=commitdiff;h=4b183bf9) the sourcing of the config file gets moved after the parsing of the options. This creates problems with the --help flag as it needs to know what BUILDSCRIPT is defined as. It also causes problems when specifying a different BUILDSCRIPT with -p as this gets set during option parsing then gets overwritten with the makepkg config file gets sourced. I don't see a nice fix for this. Sourcing the conf file before option parsing and then again after if it is changed seems not good to me. So can we back that patch out at least temporarily. Note that this does not effect maint so the 3.2.2 release will be bug free as usual. Allan

3 6

Re: [pacman-dev] makepkg: package splitting
by Miklos Vajna 10 Dec '08

10 Dec '08

On Mon, Dec 08, 2008 at 12:35:41PM +1000, Allan McRae <allan(a)archlinux.org> wrote: > pkgname=('abs-core' 'abs-readme') > pkgver=2.3 > pkgrel=1 > arch=('i686' 'x86_64') > url="http://projects.archlinux.org/git/?p=abs.git" > license=('GPL') > source=(ftp://ftp.archlinux.org/other/abs/abs-${pkgver}.tar.gz) > md5sums=('d6fd791aa487ba8bb5ff48c3ace20592') > > build() { > cd ${srcdir}/abs > make CONFDIR=/etc/ || return 1 > > # change ABS tags for x86_64 to correct values > if [ "$CARCH" = "x86_64" ]; then > sed -i "s|i686|x86_64|g" ${pkgdir}/etc/abs.conf > fi > } > > package_abs-core() { > pkgdesc="Utilities to download and work with the Arch Build System (ABS)" > depends=('bash' 'rsync') > backup=(etc/abs.conf) > > cd ${srcdir}/abs > make CONFDIR=/etc/ DESTDIR=${pkgdir} install || return 1 > } > > package_abs-readme() { > pkgdesc="ABS base directory and README file" > depends=('abs') > install=abs.install > > # Add readme file, and make base /var/abs path > install -dm0755 ${pkgdir}/var/abs/local/ > install -Dm0644 ${srcdir}/abs/README ${pkgdir}/var/abs/README > } How does this allow one to define package-dependent dependencies? Typically foo-python will have to depend on python, but foo won't. Thanks.

1 1

[pacman-dev] pacman summary!
by smurfd 10 Dec '08

10 Dec '08

Hi, i have a suggestion, like the suggestion i gave to the portage developers for the gentoo "package" manager. It would be really nice, if after a "pacman -Suy" or just a simple "pacman -S <packagelist>" that you got a summary of all the messages that has been spit out on the screen while doing the upgrade of packages. If you upgrade several packages, say 100 of them or so, it could be hard to read all the notices that every package spits out when beeing installed. i mean stuff like : *>>> Updating module dependencies. Please wait ... >>> MKINITCPIO SETUP >>> ---------------- >>> If you use LVM2, Encrypted root or software RAID, >>> Ensure you enable support in /etc/mkinitcpio.conf . >>> More information about mkinitcpio setup can be found here: >>> http://wiki.archlinux.org/index.php/Mkinitcpio >>> Generating initial ramdisk, using mkinitcpio. Please wait... * Hope you like the idea, and i dont think it should be so hard to apply. cut all of the >>> things to a temp file and after doing the last upgrade you cat whatever is in that temp file.. or something like that. Best regards /smurfd

2 2