many changes

* [breaking changes] change lcs_req_v3_t format, but it should not havea effects because there are no trace using this version
* add new lcs ver that allow ttl and features

libCacheSim/bin/cli_reader_utils.c

@@ -122,9 +122,21 @@ void parse_reader_params(const char *reader_params_str, reader_init_param_t *par
     } else if (strcasecmp(key, "tenant-col") == 0) {
       params->tenant_field = (int)(strtol(value, &end, 0));
       _check_parsed_result(end, params->tenant_field);
-    } else if (strcasecmp(key, "next-access-col") == 0) {
-      params->next_access_vtime_field = (int)strtol(value, &end, 0);
-      _check_parsed_result(end, params->next_access_vtime_field);
+    } else if (strcasecmp(key, "feature-cols") == 0) {
+      // feature-cols=1|2|3
+      char *feature_str = strdup(value);
+      char *feature = strsep(&feature_str, "|");
+      int i = 0;
+      while (feature != NULL) {
+        params->feature_fields[i] = (int)strtol(feature, &end, 0);
+        _check_parsed_result(end, params->feature_fields[i]);
+        i++;
+        feature = strsep(&feature_str, ",");
+      }
+      params->n_feature_fields = i;
+    } else if (strcasecmp(key, "ttl-col") == 0) {
+      params->ttl_field = (int)strtol(value, &end, 0);
+      _check_parsed_result(end, params->ttl_field);
     } else if (strcasecmp(key, "obj-id-is-num") == 0) {
       params->obj_id_is_num_set = true;
       params->obj_id_is_num = is_true(value);
@@ -150,7 +162,7 @@ void parse_reader_params(const char *reader_params_str, reader_init_param_t *par
             /* user input: k1=v1, delimiter=\t, k2=v2*/
             params->delimiter = '\t';
           } else if (value[1] == ',') {
-            ERROR("can I reach here?\n");
+            WARN("delimiter may be incorrect\n");
             params->delimiter = ',';
           } else {
             ERROR("unsupported delimiter: '%s'\n", value);

libCacheSim/bin/traceUtils/traceConvLCS.cpp

@@ -104,6 +104,7 @@ void convert_to_lcs(reader_t *reader, std::string ofilepath, bool output_txt, bo
     lcs_req.obj_size = req->obj_size;
     lcs_req.op = req->op;
     lcs_req.tenant = req->tenant_id;
+    lcs_req.ttl = req->ttl;
     lcs_req.next_access_vtime = req->next_access_vtime;
 
     if (lcs_req.op == OP_GET || lcs_req.op == OP_GETS || lcs_req.op == OP_READ) {
@@ -130,6 +131,9 @@ void convert_to_lcs(reader_t *reader, std::string ofilepath, bool output_txt, bo
     }
 
     ofile_temp.write(reinterpret_cast<char *>(&lcs_req), sizeof(lcs_req_full_t));
+    for (int i = 0; i < req->n_features; i++) {
+      ofile_temp.write(reinterpret_cast<char *>(&req->features[i]), sizeof(int32_t));
+    }
 
     stat.n_req_byte += req->obj_size;
     stat.n_req += 1;
@@ -312,6 +316,14 @@ static void _analyze_trace(lcs_trace_stat_t &stat, const std::unordered_map<uint
   }
 }
 
+/**
+ * @brief read the reverse trace and write to the output file
+ *
+ * @param ofilepath
+ * @param stat
+ * @param output_txt
+ * @param lcs_ver
+ */
 static void _reverse_file(std::string ofilepath, lcs_trace_stat_t stat, bool output_txt, int64_t lcs_ver) {
   size_t file_size;
   char *mapped_file = reinterpret_cast<char *>(utils::setup_mmap(ofilepath + ".reverse", &file_size));
@@ -327,8 +339,16 @@ static void _reverse_file(std::string ofilepath, lcs_trace_stat_t stat, bool out
   lcs_req_full_t lcs_req_full;
   size_t lcs_full_req_entry_size = sizeof(lcs_req_full_t);
 
-  while (pos >= lcs_full_req_entry_size) {
-    pos -= lcs_full_req_entry_size;
+  // for lcs version 4-8, we need to read the features
+  size_t n_features = 0;
+  if (lcs_ver >= 4 && lcs_ver <= 8) {
+    n_features = LCS_VER_TO_N_FEATURES[lcs_ver];
+  }
+
+  size_t entry_size = lcs_full_req_entry_size + n_features * sizeof(int32_t);
+
+  while (pos >= entry_size) {
+    pos -= entry_size;
     memcpy(&lcs_req_full, mapped_file + pos, lcs_full_req_entry_size);
     if (lcs_req_full.next_access_vtime != INT64_MAX) {
       /* req.next_access_vtime is the vtime start from the end */
@@ -363,6 +383,17 @@ static void _reverse_file(std::string ofilepath, lcs_trace_stat_t stat, bool out
       lcs_req_v3.next_access_vtime = lcs_req_full.next_access_vtime;
 
       ofile.write(reinterpret_cast<char *>(&lcs_req_v3), sizeof(lcs_req_v3));
+    } else if (lcs_ver >= 4 && lcs_ver <= 8) {
+      lcs_req_v3_t base;
+      base.clock_time = lcs_req_full.clock_time;
+      base.obj_id = lcs_req_full.obj_id;
+      base.obj_size = lcs_req_full.obj_size;
+      base.op = lcs_req_full.op;
+      base.tenant = lcs_req_full.tenant;
+      base.next_access_vtime = lcs_req_full.next_access_vtime;
+
+      ofile.write(reinterpret_cast<char *>(&base), sizeof(lcs_req_v3));
+      ofile.write(mapped_file + pos + lcs_full_req_entry_size, n_features * sizeof(int32_t));
     } else {
       ERROR("invalid lcs version %ld\n", lcs_ver);
     }

libCacheSim/bin/traceUtils/traceConvMain.cpp

@@ -15,26 +15,6 @@
  * each version has a different request struct format, however, all lcs traces have
  * the same header format which stores the version and trace statistics
  *
- * lcs_v1 is the simplest format with only clock_time, obj_id, obj_size, and next_access_vtime
- *
- * typedef struct __attribute__((packed)) lcs_req_v1 {
- *   uint32_t clock_time;
- *   uint64_t obj_id;
- *   uint32_t obj_size;
- *   int64_t next_access_vtime;
- * } lcs_req_v1_t;
- *
- *
- * lcs_v2 has more fields, operation and tenant
- *
- * typedef struct __attribute__((packed)) lcs_req_v2 {
- *   uint32_t clock_time;
- *   uint64_t obj_id;
- *   uint32_t obj_size;
- *   uint32_t op : 8;
- *   uint32_t tenant : 24;
- *   int64_t next_access_vtime;
- * } lcs_req_v2_t;
  *
  * see traceReader/generalReader/lcs.h for more details
  *
@@ -57,6 +37,16 @@ int main(int argc, char *argv[]) {
     traceConv::convert_to_lcs(args.reader, args.ofilepath, args.output_txt, args.remove_size_change, 2);
   } else if (strcasecmp(args.output_format, "lcs_v3") == 0) {
     traceConv::convert_to_lcs(args.reader, args.ofilepath, args.output_txt, args.remove_size_change, 3);
+  } else if (strcasecmp(args.output_format, "lcs_v4") == 0) {
+    traceConv::convert_to_lcs(args.reader, args.ofilepath, args.output_txt, args.remove_size_change, 4);
+  } else if (strcasecmp(args.output_format, "lcs_v5") == 0) {
+    traceConv::convert_to_lcs(args.reader, args.ofilepath, args.output_txt, args.remove_size_change, 5);
+  } else if (strcasecmp(args.output_format, "lcs_v6") == 0) {
+    traceConv::convert_to_lcs(args.reader, args.ofilepath, args.output_txt, args.remove_size_change, 6);
+  } else if (strcasecmp(args.output_format, "lcs_v7") == 0) {
+    traceConv::convert_to_lcs(args.reader, args.ofilepath, args.output_txt, args.remove_size_change, 7);
+  } else if (strcasecmp(args.output_format, "lcs_v8") == 0) {
+    traceConv::convert_to_lcs(args.reader, args.ofilepath, args.output_txt, args.remove_size_change, 8);
   } else if (strcasecmp(args.output_format, "oracleGeneral") == 0) {
     traceConv::convert_to_oracleGeneral(args.reader, args.ofilepath, args.output_txt, args.remove_size_change);
   } else {

libCacheSim/bin/traceUtils/tracePrintMain.cpp

@@ -14,16 +14,16 @@ int main(int argc, char *argv[]) {
 
   cli::parse_cmd(argc, argv, &args);
 
-  request_t *req = new_request();
-  read_one_req(args.reader, req);
-
-  bool trace_has_next_access_vtime = req->next_access_vtime != -2;
-
   if (args.print_stat) {
     lcs_print_trace_stat(args.reader);
     return 0;
   }
 
+  request_t *req = new_request();
+  read_one_req(args.reader, req);
+
+  bool trace_has_next_access_vtime = req->next_access_vtime != -2;
+
   if (!args.print_obj_id_only) {
     if (trace_has_next_access_vtime) {
       printf("# time,object,size,next_access_vtime\n");
@@ -40,8 +40,8 @@ int main(int argc, char *argv[]) {
     if (args.print_obj_id_only) {
       printf("%lu\n", (unsigned long)req->obj_id);
     } else {
-      printf("%ld%c%lu%c%d", (long)req->clock_time, args.delimiter,
-             (unsigned long)req->obj_id, args.delimiter, (int)req->obj_size);
+      printf("%ld%c%lu%c%d", (long)req->clock_time, args.delimiter, (unsigned long)req->obj_id, args.delimiter,
+             (int)req->obj_size);
       if (trace_has_next_access_vtime) {
         printf("%c%ld\n", args.delimiter, (long)req->next_access_vtime);
       } else {

libCacheSim/include/libCacheSim/reader.h

@@ -51,6 +51,9 @@ typedef struct {
   int32_t tenant_field;
   int32_t next_access_vtime_field;
 
+  int32_t n_feature_fields;
+  int32_t feature_fields[N_MAX_FEATURES];
+
   // block cache, 0 and -1 means ignore this field, 1 is also invalid
   // block_size breaks a large request for multiple blocks into multiple requests
   int32_t block_size;
@@ -154,14 +157,6 @@ static inline void set_default_reader_init_params(reader_init_param_t *params) {
   params->cap_at_n_req = -1;
   params->trace_start_offset = 0;
 
-  params->time_field = 0;
-  params->obj_id_field = 0;
-  params->obj_size_field = 0;
-  params->op_field = 0;
-  params->ttl_field = 0;
-  params->tenant_field = 0;
-  params->next_access_vtime_field = 0;
-
   params->has_header = false;
   /* whether the user has specified the has_header params */
   params->has_header_set = false;

libCacheSim/include/libCacheSim/request.h

@@ -17,11 +17,13 @@
 extern "C" {
 #endif
 
+#define N_MAX_FEATURES 16
+
 /* need to optimize this for CPU cacheline */
 typedef struct request {
   int64_t clock_time; /* use uint64_t because vscsi uses microsec timestamp */
 
-  uint64_t hv;        /* hash value, used when offloading hash to reader */
+  uint64_t hv; /* hash value, used when offloading hash to reader */
 
   /* this represents the hash of the object id in key-value cache
    * or the logical block address in block cache, note that LBA % block_size == 0 */
@@ -33,32 +35,22 @@ typedef struct request {
 
   req_op_e op;
 
+  int32_t tenant_id;
+
   uint64_t n_req;
 
   int64_t next_access_vtime;
-  /* carry necessary data between the multiple functions of serving one request
-   */
-  void *eviction_algo_data;
 
   // this is used by key-value cache traces
   struct {
     uint64_t key_size : 16;
     uint64_t val_size : 48;
   };
 
-
   int32_t ns;  // namespace
-  int32_t content_type;
-  int32_t tenant_id;
 
-  // int32_t bucket_id;
-  // int32_t age;
-  // int32_t hostname;
-  // int16_t extension;
-  // int16_t colo;
-  // int16_t n_level;
-  // int16_t n_param;
-  // int8_t method;
+  // carry necessary data between the multiple functions of serving one request
+  void *eviction_algo_data;
 
   /* used in trace analysis */
   int64_t vtime_since_last_access;
@@ -72,6 +64,10 @@ typedef struct request {
 
   bool valid; /* indicate whether request is valid request
                * it is invalid if the trace reaches the end */
+
+  int32_t n_features;
+  int32_t features[N_MAX_FEATURES];
+
 } request_t;
 
 /**