use less memory when just writing the filelist

1 files changed, 90 insertions, 17 deletions

diff --git a/src/repo_write.c b/src/repo_write.c
index 2900b44..11002b4 100644
--- a/src/repo_write.c
+++ b/src/repo_write.c
@@ -397,6 +397,7 @@ struct cbdata {
   Id lastlen;
 
   int doingsolvables;	/* working on solvables data */
+  int filelistmode;
 };
 
 #define NEEDED_BLOCK 1023
@@ -450,6 +451,19 @@ data_addideof(struct extdata *xd, Id sx, int eof)
   xd->len = dp - xd->buf;
 }
 
+static inline int
+data_addideof_len(Id sx)
+{
+  unsigned int x = (unsigned int)sx;
+  if (x >= (1 << 13))
+    {
+      if (x >= (1 << 27))
+	return 5;
+      return x >= (1 << 20) ? 4 : 3;
+    }
+  return x >= (1 << 6) ? 2 : 1;
+}
+
 static void
 data_addid64(struct extdata *xd, unsigned int x, unsigned int hx)
 {
@@ -861,8 +875,15 @@ repo_write_adddata(struct cbdata *cbdata, Repodata *data, Repokey *key, KeyValue
 	if (cbdata->owndirpool)
 	  id = putinowndirpool(cbdata, data, &data->dirpool, id);
 	id = cbdata->dirused[id];
+	if (cbdata->filelistmode > 0)
+	  {
+	    xd->len += data_addideof_len(id) + strlen(kv->str) + 1;
+	    break;
+	  }
 	data_addideof(xd, id, kv->eof);
 	data_addblob(xd, (unsigned char *)kv->str, strlen(kv->str) + 1);
+	if (cbdata->filelistmode < 0)
+	  return 0;
 	break;
       case REPOKEY_TYPE_FIXARRAY:
 	if (kv->eof == 0)
@@ -1032,6 +1053,30 @@ is_filelist_extension(Repodata *data)
   return 1;
 }
 
+
+static int
+write_compressed_extdata(Repodata *target, struct extdata *xd, unsigned char *vpage, int lpage)
+{
+  unsigned char *dp = xd->buf;
+  int l = xd->len;
+  while (l)
+    {
+      int ll = REPOPAGE_BLOBSIZE - lpage;
+      if (l < ll)
+	ll = l;
+      memcpy(vpage + lpage, dp, ll);
+      dp += ll;
+      lpage += ll;
+      l -= ll;
+      if (lpage == REPOPAGE_BLOBSIZE)
+	{
+	  write_compressed_page(target, vpage, lpage);
+	  lpage = 0;
+	}
+    }
+  return lpage;
+}
+
 /*
  * Repo
  */
@@ -1731,6 +1776,19 @@ fprintf(stderr, "dir %d used %d\n", i, cbdata.dirused ? cbdata.dirused[i] : 1);
     {
       data_addid(xd, repo->nsolvables);	/* FLEXARRAY nentries */
       cbdata.doingsolvables = 1;
+
+      /* check if we can do the special filelist memory optimization */
+      if (anyrepodataused)
+	{
+	  for (i = 1; i < target.nkeys; i++)
+	    if (target.keys[i].storage == KEY_STORAGE_VERTICAL_OFFSET)
+	      cbdata.filelistmode |= cbdata.filelistmode == 0 && target.keys[i].type == REPOKEY_TYPE_DIRSTRARRAY ? 1 : 2;
+	    else if (target.keys[i].type == REPOKEY_TYPE_DIRSTRARRAY)
+	      cbdata.filelistmode = 2;
+	  if (cbdata.filelistmode != 1)
+	    cbdata.filelistmode = 0;
+	}
+
       for (i = repo->start, s = pool->solvables + i, n = 0; i < repo->end; i++, s++)
 	{
 	  if (s->repo != repo)
@@ -1918,31 +1976,46 @@ fprintf(stderr, "dir %d used %d\n", i, cbdata.dirused ? cbdata.dirused[i] : 1);
   if (i < target.nkeys)
     {
       /* yes, write it in pages */
-      unsigned char *dp, vpage[REPOPAGE_BLOBSIZE];
-      int l, ll, lpage = 0;
+      unsigned char vpage[REPOPAGE_BLOBSIZE];
+      int lpage = 0;
 
       write_u32(&target, REPOPAGE_BLOBSIZE);
       for (i = 1; i < target.nkeys; i++)
+	if (cbdata.extdata[i].len)
+	  {
+	    if (cbdata.filelistmode)
+	      break;
+	    lpage = write_compressed_extdata(&target, cbdata.extdata + i, vpage, lpage);
+	  }
+      if (cbdata.filelistmode && i < target.nkeys)
 	{
-	  if (!cbdata.extdata[i].len)
-	    continue;
-	  l = cbdata.extdata[i].len;
-	  dp = cbdata.extdata[i].buf;
-	  while (l)
+	  /* ok, just this single extdata, which is a filelist */
+	  xd = cbdata.extdata + i;
+	  xd->len = 0;
+	  cbdata.filelistmode = -1;
+	  for (j = 0; j < cbdata.nkeymap; j++)
+	    if (cbdata.keymap[j] != i)
+	      cbdata.keymap[j] = 0;
+	  for (i = repo->start, s = pool->solvables + i; i < repo->end; i++, s++)
 	    {
-	      ll = REPOPAGE_BLOBSIZE - lpage;
-	      if (l < ll)
-		ll = l;
-	      memcpy(vpage + lpage, dp, ll);
-	      dp += ll;
-	      lpage += ll;
-	      l -= ll;
-	      if (lpage == REPOPAGE_BLOBSIZE)
+	      if (s->repo != repo)
+		continue;
+	      FOR_REPODATAS(repo, j, data)
+		{
+		  if (!repodataused[j])
+		    continue;
+		  if (i < data->start || i >= data->end)
+		    continue;
+		  repodata_search(data, i, 0, SEARCH_SUB|SEARCH_ARRAYSENTINEL, repo_write_cb_adddata, &cbdata);
+		}
+	      if (xd->len > 1024 * 1024)
 		{
-		  write_compressed_page(&target, vpage, lpage);
-		  lpage = 0;
+		  lpage = write_compressed_extdata(&target, xd, vpage, lpage);
+		  xd->len = 0;
 		}
 	    }
+	  if (xd->len)
+	    lpage = write_compressed_extdata(&target, xd, vpage, lpage);
 	}
       if (lpage)
 	write_compressed_page(&target, vpage, lpage);