remove SECTOR_SIZE and msr traces need to be processed again;

fix bugs in traceConv: version issue, WRITE is not counted

libCacheSim/bin/traceUtils/traceConvLCS.cpp

@@ -108,7 +108,8 @@ void convert_to_lcs(reader_t *reader, std::string ofilepath, bool output_txt, bo
 
     if (lcs_req.op == OP_GET || lcs_req.op == OP_GETS || lcs_req.op == OP_READ) {
       stat.n_read++;
-    } else if (lcs_req.op == OP_SET || lcs_req.op == OP_REPLACE || lcs_req.op == OP_ADD || lcs_req.op == OP_UPDATE) {
+    } else if (lcs_req.op == OP_WRITE || lcs_req.op == OP_SET || lcs_req.op == OP_REPLACE || lcs_req.op == OP_ADD ||
+               lcs_req.op == OP_UPDATE) {
       stat.n_write++;
     } else if (lcs_req.op == OP_DELETE) {
       stat.n_delete++;

scripts/lcs_reader.py

@@ -159,7 +159,7 @@ def parse_stat(b, print_stat=True):
         print()
 
 
-def read_header(ifile):
+def read_header(ifile, print_stat=True):
     header = ifile.read(LCS_HEADER_SIZE)
     start_magic, version = struct.unpack("<QQ", header[:16])
     end_magic = struct.unpack("<Q", header[-8:])[0]
@@ -168,20 +168,21 @@ def read_header(ifile):
     if end_magic != LCS_END_MAGIC:
         raise RuntimeError(f"Invalid trace file end magic {end_magic:016x}")
 
-    parse_stat(header[16:-176], print_stat=True)
+    parse_stat(header[16:-176], print_stat=print_stat)
 
     return version
 
 
 def read_trace(ifilepath, n_max_req=-1):
     if ifilepath.endswith(".zst"):
         import zstandard as zstd
+
         decompressor = zstd.ZstdDecompressor()
         reader = decompressor.stream_reader(open(ifilepath, "rb"))
-    else:            
+    else:
         ifile = open(ifilepath, "rb")
         reader = ifile
-        
+
     version = read_header(reader)
     s = [
         struct.Struct("<IQIq"),
@@ -204,11 +205,43 @@ def read_trace(ifilepath, n_max_req=-1):
     reader.close()
 
 
+def test_block_trace(ifilepath):
+    if ifilepath.endswith(".zst"):
+        import zstandard as zstd
+
+        decompressor = zstd.ZstdDecompressor()
+        reader = decompressor.stream_reader(open(ifilepath, "rb"))
+    else:
+        ifile = open(ifilepath, "rb")
+        reader = ifile
+
+    version = read_header(reader, print_stat=False)
+    s = [
+        struct.Struct("<IQIq"),
+        struct.Struct("<IQIIq"),
+        struct.Struct("<qQqIq"),
+    ][version - 1]
+
+    while True:
+        b = reader.read(s.size)
+        if not b:
+            break
+        req = s.unpack(b)
+        if req[1] % 4096 != 0:
+            raise RuntimeError(
+                f"lba is not multiple of block size {req[1]} % 4096 = {req[1] % 4096} {ifilepath}"
+            )
+
+    reader.close()
+    print(f"LBA test passed {ifilepath}")
+
+
 if __name__ == "__main__":
     import sys
 
     if len(sys.argv) < 2:
         print(f"Usage: {sys.argv[0]} /path/trace [n_req]")
         sys.exit(1)
 
-    read_trace(sys.argv[1], int(sys.argv[2]) if len(sys.argv) > 2 else -1)
+    test_block_trace(sys.argv[1])
+    # read_trace(sys.argv[1], int(sys.argv[2]) if len(sys.argv) > 2 else -1)

scripts/traceConv/alibabaBlock.py

@@ -21,17 +21,22 @@
 
 ###############################
 
-
-# this is used to convert lbn to lba
-SECTOR_SIZE = 512
 # this is used to convert requests to multiple 4K blocks
 BLOCK_SIZE = 4096
 
+# because each volume may access the same LBA, we add MAX_VOL_SIZE to lba to make it unique
+MAX_VOL_SIZE = 100 * 1024 * 1024 * 1024 * 1024  # 100TiB
 
 def preprocess(ifilepath, ofilepath=None):
-    """we preprocess the trace into a csv format with only necessary information"""
+    """
+    preprocess the trace into a csv format with only necessary information
+    this step aims to normalize the trace format before converting it to lcs format
+
+    """
+
     if os.path.exists(ifilepath + ".stat"):
         return
+
     if not ofilepath:
         ofilepath = ifilepath + ".pre_lcs"
     ifile = open(ifilepath, "r")
@@ -49,8 +54,12 @@ def preprocess(ifilepath, ofilepath=None):
 
         vol_id, op, offset, req_size, ts = parts
 
-        ts = int(ts)//1_000_000
+        ts = int(ts) // 1_000_000
         lba = int(offset)
+        # because different volumes may access the same LBA
+        # we add volume id to lba to make it unique
+        lba += int(vol_id) * MAX_VOL_SIZE
+
         req_size = int(req_size)
         if op == "R":
             op = "read"
@@ -61,12 +70,8 @@ def preprocess(ifilepath, ofilepath=None):
         else:
             raise RuntimeError(f"Unknown operation: {op} {req_size} {lba} {ts}")
 
-        assert (
-            req_size % SECTOR_SIZE == 0
-        ), "req_size is not multiple of sector size {}%{}".format(req_size, SECTOR_SIZE)
-        assert (
-            lba % SECTOR_SIZE == 0
-        ), "lba is not multiple of sector size {}%{}".format(lba, SECTOR_SIZE)
+        # align lba to block size to BLOCK_SIZE
+        lba = lba - (lba % BLOCK_SIZE)
 
         if not start_ts:
             start_ts = ts
@@ -105,29 +110,31 @@ def preprocess(ifilepath, ofilepath=None):
 
 def convert(traceConv_path, ifilepath, ofilepath=None):
     if not ofilepath:
-        ofilepath = ifilepath.replace(".pre_lcs", "") + ".lcs"
+        ofilepath = ifilepath.replace(".pre_lcs", ".lcs")
     p = subprocess.run(
-        f'{traceConv_path} {ifilepath} csv -t "time-col=1,obj-id-col=2,obj-size-col=3,op-col=4" -o {ofilepath}',
+        f'{traceConv_path} {ifilepath} csv -t "time-col=1,obj-id-col=2,obj-size-col=3,op-col=4,obj-id-is-num=1" -o {ofilepath} --output-format lcs_v2',
         shell=True,
     )
-    print(p.returncode)
-    print(p.stdout.decode())
-    print(p.stderr.decode())
-    print(f"Converted trace is saved to {ofilepath}")
+    if p.returncode == 0:
+        print(f"Converted trace is saved to {ofilepath}")
 
 
 if __name__ == "__main__":
+    from utils import post_process
+    DEFAULT_TRACECONV_PATH = BASEPATH + "/_build/bin/traceConv"
+
     if len(sys.argv) < 2:
         print("Usage: {} <trace file>".format(sys.argv[0]))
         sys.exit(1)
 
     ifilepath = sys.argv[1]
+    traceConv_path = os.environ.get("TRACECONV_PATH", DEFAULT_TRACECONV_PATH)
+
     try:
-        preprocess(ifilepath)
+        preprocess(ifilepath, ifilepath + ".pre_lcs")
+        convert(traceConv_path, ifilepath + ".pre_lcs", ofilepath=ifilepath + ".lcs")
+        post_process(ifilepath)
     except Exception as e:
-        os.remove(ifilepath + ".pre_lcs")
         print(e)
         with open(ifilepath + ".fail", "w") as f:
             f.write(str(e))
-
-    # convert(BASEPATH + "/_build/bin/traceConv", ifilepath)

scripts/traceConv/cloudphysics.py

@@ -6,6 +6,7 @@
 import subprocess
 from math import ceil
 
+
 CURRFILE_PATH = os.path.dirname(os.path.abspath(__file__))
 BASEPATH = os.path.join(CURRFILE_PATH, "..", "..")
 sys.path.append(BASEPATH)
@@ -93,11 +94,10 @@ def find_version_method2(ifilepath, n_test=800):
 
 
 def preprocess(ifilepath, ofilepath=None):
-    """we preprocess the trace into a csv format with only necessary information
+    """
+    preprocess the trace into a csv format with only necessary information
+    this step aims to normalize the trace format before converting it to lcs format
 
-    Args:
-        ifilepath (str): input trace file path
-        ofilepath (str, optional): output path. Defaults to ifilepath.pre_lcs
     """
 
     if os.path.exists(ifilepath + ".stat"):
@@ -165,13 +165,10 @@ def preprocess(ifilepath, ofilepath=None):
         else:
             raise RuntimeError(f"Unknown operation: {cmd} {req_size} {lbn} {ts}")
 
+        # align lba to block size to BLOCK_SIZE
+        lba = lba - (lba % BLOCK_SIZE)
+
         # write to file
-        assert (
-            req_size % SECTOR_SIZE == 0
-        ), "req_size is not multiple of sector size {}%{}".format(req_size, SECTOR_SIZE)
-        assert (
-            lba % SECTOR_SIZE == 0
-        ), "lba is not multiple of sector size {}%{}".format(lba, SECTOR_SIZE)
         for i in range(int(ceil(req_size / BLOCK_SIZE))):
             ofile.write(
                 "{},{},{},{}\n".format(ts, lba + i * BLOCK_SIZE, BLOCK_SIZE, op)
@@ -205,30 +202,31 @@ def preprocess(ifilepath, ofilepath=None):
 
 def convert(traceConv_path, ifilepath, ofilepath=None):
     if not ofilepath:
-        ofilepath = ifilepath.replace(".pre_lcs", "") + ".lcs"
+        ofilepath = ifilepath.replace(".pre_lcs", ".lcs")
     p = subprocess.run(
-        f'{traceConv_path} {ifilepath} csv -t "time-col=1,obj-id-col=2,obj-size-col=3,op-col=4" -o {ofilepath}',
+        f'{traceConv_path} {ifilepath} csv -t "time-col=1,obj-id-col=2,obj-size-col=3,op-col=4,obj-id-is-num=1" -o {ofilepath} --output-format lcs_v2',
         shell=True,
     )
-    print(p.returncode)
-    print(p.stdout.decode())
-    print(p.stderr.decode())
-    print(f"Converted trace is saved to {ofilepath}")
+    if p.returncode == 0:
+        print(f"Converted trace is saved to {ofilepath}")
 
 
 if __name__ == "__main__":
+    from utils import post_process
+    DEFAULT_TRACECONV_PATH = BASEPATH + "/_build/bin/traceConv"
+
     if len(sys.argv) < 2:
         print("Usage: {} <trace file>".format(sys.argv[0]))
         sys.exit(1)
 
     ifilepath = sys.argv[1]
-    # ifilepath = "/disk/anonymized106/w104_vscsi1.vscsitrace"
+    traceConv_path = os.environ.get("TRACECONV_PATH", DEFAULT_TRACECONV_PATH)
+
     try:
-        preprocess(ifilepath)
+        preprocess(ifilepath, ifilepath + ".pre_lcs")
+        convert(traceConv_path, ifilepath + ".pre_lcs", ofilepath=ifilepath + ".lcs")
+        post_process(ifilepath)
     except Exception as e:
-        os.remove(ifilepath + ".pre_lcs")
         print(e)
         with open(ifilepath + ".fail", "w") as f:
             f.write(str(e))
-
-    # convert(BASEPATH + "/_build/bin/traceConv", ifilepath)

scripts/traceConv/msr.py

@@ -19,23 +19,19 @@
 ###############################
 
 
-# this is used to convert lbn to lba
-SECTOR_SIZE = 512
 # this is used to convert requests to multiple 4K blocks
-BLOCK_SIZE = 4096
+BLOCK_SIZE = 4096 * 16
 
 
 def preprocess(ifilepath, ofilepath=None):
-    """we preprocess the trace into a csv format with only necessary information
+    """preprocess the trace into a csv format with only necessary information
+    this step aims to normalize the trace format before converting it to lcs format
 
-    Args:
-        ifilepath (_type_): _description_
-        ofilepath (_type_, optional): _description_. Defaults to None.
     """
-    # start_time = time.time()
+    start_time = time.time()
 
-    # if os.path.exists(ifilepath + ".stat"):
-    #     return
+    if os.path.exists(ifilepath + ".stat"):
+        return
 
     if not ofilepath:
         ofilepath = ifilepath + ".pre_lcs"
@@ -59,10 +55,6 @@ def preprocess(ifilepath, ofilepath=None):
             start_ts = ts
         end_ts = ts
         n_original_req += 1
-        # if n_original_req % 100000 == 0:
-        #     print(
-        #         f"{time.time()-start_time:8.2f}sec: {n_original_req/100000:.2f}, {n_req/100000:.2f}, {len(seen_blocks)/100000:.2f}"
-        #     )
 
         lba = int(offset)
         req_size = int(req_size)
@@ -75,13 +67,8 @@ def preprocess(ifilepath, ofilepath=None):
         else:
             print("Unknown operation: {}".format(op))
 
-        # write to file
-        assert (
-            req_size % SECTOR_SIZE == 0
-        ), "req_size is not multiple of sector size {}%{}".format(req_size, SECTOR_SIZE)
-        assert (
-            lba % SECTOR_SIZE == 0
-        ), "lba is not multiple of sector size {}%{}".format(lba, SECTOR_SIZE)
+        # align lba to block size to BLOCK_SIZE
+        lba = lba - (lba % BLOCK_SIZE)
 
         for i in range(int(ceil(req_size / BLOCK_SIZE))):
             ofile.write(
@@ -118,29 +105,31 @@ def preprocess(ifilepath, ofilepath=None):
 
 def convert(traceConv_path, ifilepath, ofilepath=None):
     if not ofilepath:
-        ofilepath = ifilepath.replace(".pre_lcs", "") + ".lcs"
+        ofilepath = ifilepath.replace(".pre_lcs", ".lcs")
     p = subprocess.run(
-        f'{traceConv_path} {ifilepath} csv -t "time-col=1,obj-id-col=2,obj-size-col=3,op-col=4" -o {ofilepath}',
+        f'{traceConv_path} {ifilepath} csv -t "time-col=1,obj-id-col=2,obj-size-col=3,op-col=4,obj-id-is-num=1" -o {ofilepath} --output-format lcs_v2',
         shell=True,
     )
-    print(p.returncode)
-    print(p.stdout.decode())
-    print(p.stderr.decode())
-    print(f"Converted trace is saved to {ofilepath}")
+    if p.returncode == 0:
+        print(f"Converted trace is saved to {ofilepath}")
 
 
 if __name__ == "__main__":
+    from utils import post_process
+    DEFAULT_TRACECONV_PATH = BASEPATH + "/_build/bin/traceConv"
+
     if len(sys.argv) < 2:
         print("Usage: {} <trace file>".format(sys.argv[0]))
         sys.exit(1)
 
     ifilepath = sys.argv[1]
+    traceConv_path = os.environ.get("TRACECONV_PATH", DEFAULT_TRACECONV_PATH)
+
     try:
-        preprocess(ifilepath)
+        preprocess(ifilepath, ifilepath + ".pre_lcs")
+        convert(traceConv_path, ifilepath + ".pre_lcs", ofilepath=ifilepath + ".lcs")
+        post_process(ifilepath)
     except Exception as e:
-        os.remove(ifilepath + ".pre_lcs")
         print(e)
         with open(ifilepath + ".fail", "w") as f:
             f.write(str(e))
-
-    # convert(BASEPATH + "/_build/bin/traceConv", ifilepath)