[ Dec 21 00:47:34 Enabled. ]
[ Dec 21 00:47:34 Rereading configuration. ]
[ Dec 21 00:47:36 Rereading configuration. ]
[ Dec 21 00:47:40 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_590ffd4f-3c08-4aaa-9320-324eceef269f/crucible -l [fd00:1122:3344:101::12]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_590ffd4f-3c08-4aaa-9320-324eceef269f/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:47:40.430489597Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439}
{"msg":"listen IP: [fd00:1122:3344:101::12]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:47:40.433739387Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:47:40.433749647Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:47:40.450165611Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:47:40.553662282Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:08.690789374Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:54636"}
{"msg":"region 3d93ccd1-6827-47ba-b053-605b0ed644d1 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:08.72238203Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:08.726430065Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:08.731411208Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"uri":"/crucible/0/regions","method":"POST","req_id":"4aae7324-ccc6-4f60-9871-9c139e320bed","remote_addr":"[fd00:1122:3344:101::a]:54636","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":12956,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_590ffd4f-3c08-4aaa-9320-324eceef269f/crucible/regions/3d93ccd1-6827-47ba-b053-605b0ed644d1","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:08.73801543Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_590ffd4f-3c08-4aaa-9320-324eceef269f/crucible/regions/3d93ccd1-6827-47ba-b053-605b0ed644d1","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:08.73803829Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"3d93ccd1-6827-47ba-b053-605b0ed644d1\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/3d93ccd1-6827-47ba-b053-605b0ed644d1\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:08.830604858Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"region":"3d93ccd1-6827-47ba-b053-605b0ed644d1","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"3d93ccd1-6827-47ba-b053-605b0ed644d1\" \"--data\" \"/data/regions/3d93ccd1-6827-47ba-b053-605b0ed644d1\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:08.832674316Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"region":"3d93ccd1-6827-47ba-b053-605b0ed644d1","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:08.925776924Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"uri":"/crucible/0/regions","method":"POST","req_id":"32fb304b-1184-485a-8f4e-f9b54168f072","remote_addr":"[fd00:1122:3344:101::a]:54636","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":88,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:09.075285459Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"region":"3d93ccd1-6827-47ba-b053-605b0ed644d1","component":"worker"}
{"msg":"region 3d93ccd1-6827-47ba-b053-605b0ed644d1 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:09.07533133Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:09.075436201Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-3d93ccd1-6827-47ba-b053-605b0ed644d1","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:09.07586378Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-3d93ccd1-6827-47ba-b053-605b0ed644d1","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:09.082263718Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:09.09366193Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-3d93ccd1-6827-47ba-b053-605b0ed644d1","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:09.097713025Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3d93ccd1-6827-47ba-b053-605b0ed644d1","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:09.09849428Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:09.098558901Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:09.098604331Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:09.098678953Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:09.104759456Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:09.112160443Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:09.575110834Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"uri":"/crucible/0/regions","method":"POST","req_id":"225d9dc7-0662-4497-8c57-59c929f48300","remote_addr":"[fd00:1122:3344:101::a]:54636","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.348199616Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:37049"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.352750351Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"uri":"/crucible/0/regions/3d93ccd1-6827-47ba-b053-605b0ed644d1","method":"GET","req_id":"5432b1dc-6fe2-492c-bd64-4f780f7520d1","remote_addr":"[fd00:1122:3344:101::a]:37049","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":4502,"response_code":"200"}
{"msg":"path is \"/data/regions/3d93ccd1-6827-47ba-b053-605b0ed644d1\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.35595755Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"datafile"}
{"msg":"dataset is oxp_590ffd4f-3c08-4aaa-9320-324eceef269f/crucible/regions/3d93ccd1-6827-47ba-b053-605b0ed644d1","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.365445196Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"datafile"}
{"msg":"snapshot is oxp_590ffd4f-3c08-4aaa-9320-324eceef269f/crucible/regions/3d93ccd1-6827-47ba-b053-605b0ed644d1@99d9a509-237d-43cd-9c13-86aab0a26b82","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.373332113Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_590ffd4f-3c08-4aaa-9320-324eceef269f/crucible/regions/3d93ccd1-6827-47ba-b053-605b0ed644d1\", \"99d9a509-237d-43cd-9c13-86aab0a26b82\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.373363393Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.387112038Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"uri":"/crucible/0/regions/3d93ccd1-6827-47ba-b053-605b0ed644d1/snapshots/99d9a509-237d-43cd-9c13-86aab0a26b82","method":"GET","req_id":"645514c7-bdf1-40a0-8a38-b611bac13697","remote_addr":"[fd00:1122:3344:101::a]:37049","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":31161,"response_code":"200"}
{"msg":"path is \"/data/regions/3d93ccd1-6827-47ba-b053-605b0ed644d1\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.391036761Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"datafile"}
{"msg":"dataset is oxp_590ffd4f-3c08-4aaa-9320-324eceef269f/crucible/regions/3d93ccd1-6827-47ba-b053-605b0ed644d1","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.399024979Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"datafile"}
{"msg":"snapshot is oxp_590ffd4f-3c08-4aaa-9320-324eceef269f/crucible/regions/3d93ccd1-6827-47ba-b053-605b0ed644d1@99d9a509-237d-43cd-9c13-86aab0a26b82","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.407243832Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_590ffd4f-3c08-4aaa-9320-324eceef269f/crucible/regions/3d93ccd1-6827-47ba-b053-605b0ed644d1\", \"99d9a509-237d-43cd-9c13-86aab0a26b82\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.407283743Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 3d93ccd1-6827-47ba-b053-605b0ed644d1-99d9a509-237d-43cd-9c13-86aab0a26b82 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.41629799Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"datafile"}
{"msg":"applying SMF actions for region 3d93ccd1-6827-47ba-b053-605b0ed644d1 running snapshot 99d9a509-237d-43cd-9c13-86aab0a26b82 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.420810223Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.420871085Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"uri":"/crucible/0/regions/3d93ccd1-6827-47ba-b053-605b0ed644d1/snapshots/99d9a509-237d-43cd-9c13-86aab0a26b82/run","method":"POST","req_id":"36354967-2fe7-4b76-b354-b4bb2f769fa8","remote_addr":"[fd00:1122:3344:101::a]:37049","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":29868,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-3d93ccd1-6827-47ba-b053-605b0ed644d1-99d9a509-237d-43cd-9c13-86aab0a26b82","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.430449852Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-3d93ccd1-6827-47ba-b053-605b0ed644d1-99d9a509-237d-43cd-9c13-86aab0a26b82","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.438710276Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-3d93ccd1-6827-47ba-b053-605b0ed644d1-99d9a509-237d-43cd-9c13-86aab0a26b82","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.464501324Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3d93ccd1-6827-47ba-b053-605b0ed644d1/.zfs/snapshot/99d9a509-237d-43cd-9c13-86aab0a26b82","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.46478689Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.464850871Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.464894372Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.464935882Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.464985953Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.473920679Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.487146824Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"worker"}
{"msg":"region 3d93ccd1-6827-47ba-b053-605b0ed644d1 running snapshot 99d9a509-237d-43cd-9c13-86aab0a26b82 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.487165935Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"datafile"}
{"msg":"path is \"/data/regions/3d93ccd1-6827-47ba-b053-605b0ed644d1\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.594671639Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"datafile"}
{"msg":"dataset is oxp_590ffd4f-3c08-4aaa-9320-324eceef269f/crucible/regions/3d93ccd1-6827-47ba-b053-605b0ed644d1","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.603135937Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"datafile"}
{"msg":"snapshot is oxp_590ffd4f-3c08-4aaa-9320-324eceef269f/crucible/regions/3d93ccd1-6827-47ba-b053-605b0ed644d1@99d9a509-237d-43cd-9c13-86aab0a26b82","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.611453861Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_590ffd4f-3c08-4aaa-9320-324eceef269f/crucible/regions/3d93ccd1-6827-47ba-b053-605b0ed644d1\", \"99d9a509-237d-43cd-9c13-86aab0a26b82\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.611485071Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.620667652Z","hostname":"oxz_crucible_53889989-d09a-4ae2-a9ba-7db0bf231073","pid":26439,"uri":"/crucible/0/regions/3d93ccd1-6827-47ba-b053-605b0ed644d1/snapshots/99d9a509-237d-43cd-9c13-86aab0a26b82/run","method":"POST","req_id":"02872fb9-e11c-4e71-bcfb-b12c617ee4fd","remote_addr":"[fd00:1122:3344:101::a]:37049","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":26476,"response_code":"200"}