[ Nov 21 19:33:05 Enabled. ]
[ Nov 21 19:33:05 Rereading configuration. ]
[ Nov 21 19:33:06 Rereading configuration. ]
[ Nov 21 19:33:10 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_784212b9-62b7-432d-b0d7-e4600fa06921/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_784212b9-62b7-432d-b0d7-e4600fa06921/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:10.162781207Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:10.164272427Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:10.164281568Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:10.183488484Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:10.272421982Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:39.234363867Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:57652"}
{"msg":"region 6c87e384-8bf8-4e18-9083-ffcddd13dbe3 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:39.267211086Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:39.281588438Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:39.286490633Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"uri":"/crucible/0/regions","method":"POST","req_id":"aae24acb-772f-4fad-aaee-70b1f566f505","remote_addr":"[fd00:1122:3344:101::b]:57652","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":24992,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_784212b9-62b7-432d-b0d7-e4600fa06921/crucible/regions/6c87e384-8bf8-4e18-9083-ffcddd13dbe3","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:39.291724713Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_784212b9-62b7-432d-b0d7-e4600fa06921/crucible/regions/6c87e384-8bf8-4e18-9083-ffcddd13dbe3","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:39.291747354Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"6c87e384-8bf8-4e18-9083-ffcddd13dbe3\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/6c87e384-8bf8-4e18-9083-ffcddd13dbe3\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:39.538600341Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"region":"6c87e384-8bf8-4e18-9083-ffcddd13dbe3","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"6c87e384-8bf8-4e18-9083-ffcddd13dbe3\" \"--data\" \"/data/regions/6c87e384-8bf8-4e18-9083-ffcddd13dbe3\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:39.540701719Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"region":"6c87e384-8bf8-4e18-9083-ffcddd13dbe3","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:39.596219281Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"uri":"/crucible/0/regions","method":"POST","req_id":"9b94a73f-6d4c-441e-b9e7-b1cbb831e206","remote_addr":"[fd00:1122:3344:101::b]:57652","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":85,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:40.021522314Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"region":"6c87e384-8bf8-4e18-9083-ffcddd13dbe3","component":"worker"}
{"msg":"region 6c87e384-8bf8-4e18-9083-ffcddd13dbe3 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:40.021575414Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:40.021669856Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-6c87e384-8bf8-4e18-9083-ffcddd13dbe3","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:40.022051711Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:40.06387313Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"uri":"/crucible/0/regions","method":"POST","req_id":"13ea3fef-821f-482b-a0dd-5b77b1cedade","remote_addr":"[fd00:1122:3344:101::b]:57652","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":66,"response_code":"200"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-6c87e384-8bf8-4e18-9083-ffcddd13dbe3","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:40.085818663Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:40.200425584Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-6c87e384-8bf8-4e18-9083-ffcddd13dbe3","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:40.232102847Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6c87e384-8bf8-4e18-9083-ffcddd13dbe3","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:40.232867518Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:40.232932689Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:40.232975689Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:40.23301356Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:40.264258577Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:40.300060346Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:42.356662664Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"uri":"/crucible/0/regions/6c87e384-8bf8-4e18-9083-ffcddd13dbe3","method":"GET","req_id":"4c9109fd-c2f5-4f69-a427-a8448b7d050b","remote_addr":"[fd00:1122:3344:101::b]:57652","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":86,"response_code":"200"}
{"msg":"path is \"/data/regions/6c87e384-8bf8-4e18-9083-ffcddd13dbe3\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:42.360006948Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"datafile"}
{"msg":"dataset is oxp_784212b9-62b7-432d-b0d7-e4600fa06921/crucible/regions/6c87e384-8bf8-4e18-9083-ffcddd13dbe3","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:42.371448011Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"datafile"}
{"msg":"region 6c87e384-8bf8-4e18-9083-ffcddd13dbe3 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:42.381730059Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:42.381888521Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:42.381957402Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"uri":"/crucible/0/regions/6c87e384-8bf8-4e18-9083-ffcddd13dbe3","method":"DELETE","req_id":"a2dfe133-f0b4-4a84-b936-9ff7c132da9a","remote_addr":"[fd00:1122:3344:101::b]:57652","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":21973,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-6c87e384-8bf8-4e18-9083-ffcddd13dbe3 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:42.385090313Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:42.385357997Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"uri":"/crucible/0/regions/6c87e384-8bf8-4e18-9083-ffcddd13dbe3","method":"GET","req_id":"8068315c-e82f-4aee-89ba-af22dd716909","remote_addr":"[fd00:1122:3344:101::b]:57652","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":26,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:42.423694339Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_784212b9-62b7-432d-b0d7-e4600fa06921/crucible/regions/6c87e384-8bf8-4e18-9083-ffcddd13dbe3\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:42.436330068Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"region":"6c87e384-8bf8-4e18-9083-ffcddd13dbe3","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:42.532692355Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"uri":"/crucible/0/regions/6c87e384-8bf8-4e18-9083-ffcddd13dbe3","method":"GET","req_id":"0d7b11cf-14d8-4425-be64-613485559fdc","remote_addr":"[fd00:1122:3344:101::b]:57652","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:42.679915562Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"uri":"/crucible/0/regions/6c87e384-8bf8-4e18-9083-ffcddd13dbe3","method":"GET","req_id":"25cbdbcd-9b35-4dd7-b6a1-2a0379cb3994","remote_addr":"[fd00:1122:3344:101::b]:57652","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":34,"response_code":"200"}
{"msg":"region 6c87e384-8bf8-4e18-9083-ffcddd13dbe3 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:42.717317122Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:33:42.794062667Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"uri":"/crucible/0/regions/6c87e384-8bf8-4e18-9083-ffcddd13dbe3","method":"GET","req_id":"187673ab-29b8-4e5f-9380-77221298ad5d","remote_addr":"[fd00:1122:3344:101::b]:57652","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":42,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:34:19.923868851Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:38818"}
{"msg":"region 9e71e860-2560-46f3-bfb7-9fc4638fa7ab state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:34:19.924047504Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:34:19.924226606Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"uri":"/crucible/0/regions","method":"POST","req_id":"403144c8-6cce-4171-a9b5-b1b1637eaa0a","remote_addr":"[fd00:1122:3344:101::a]:38818","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":223,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:34:19.924248046Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_784212b9-62b7-432d-b0d7-e4600fa06921/crucible/regions/9e71e860-2560-46f3-bfb7-9fc4638fa7ab","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:34:19.93421789Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_784212b9-62b7-432d-b0d7-e4600fa06921/crucible/regions/9e71e860-2560-46f3-bfb7-9fc4638fa7ab","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:34:19.93425756Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"9e71e860-2560-46f3-bfb7-9fc4638fa7ab\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/9e71e860-2560-46f3-bfb7-9fc4638fa7ab\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:34:19.993236908Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"region":"9e71e860-2560-46f3-bfb7-9fc4638fa7ab","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"9e71e860-2560-46f3-bfb7-9fc4638fa7ab\" \"--data\" \"/data/regions/9e71e860-2560-46f3-bfb7-9fc4638fa7ab\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:34:19.993289249Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"region":"9e71e860-2560-46f3-bfb7-9fc4638fa7ab","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:34:20.120281196Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"region":"9e71e860-2560-46f3-bfb7-9fc4638fa7ab","component":"worker"}
{"msg":"region 9e71e860-2560-46f3-bfb7-9fc4638fa7ab state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:34:20.120334726Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:34:20.120402857Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-9e71e860-2560-46f3-bfb7-9fc4638fa7ab","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:34:20.121816626Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-9e71e860-2560-46f3-bfb7-9fc4638fa7ab","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:34:20.131620117Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:34:20.149183242Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-9e71e860-2560-46f3-bfb7-9fc4638fa7ab","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:34:20.153883575Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/9e71e860-2560-46f3-bfb7-9fc4638fa7ab","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:34:20.154120458Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:34:20.154174209Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:34:20.154213559Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:34:20.15425242Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:34:20.161995373Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:34:20.17221465Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:34:20.294820688Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"uri":"/crucible/0/regions","method":"POST","req_id":"5ca0cb05-aa01-4a6c-92e3-e10502223e98","remote_addr":"[fd00:1122:3344:101::a]:38818","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":63,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:51.511073816Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:58437"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:51.511205217Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"uri":"/crucible/0/regions/9e71e860-2560-46f3-bfb7-9fc4638fa7ab","method":"GET","req_id":"59e587a4-c3ec-4b29-bd39-d09ec8e85734","remote_addr":"[fd00:1122:3344:101::a]:58437","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":51,"response_code":"200"}
{"msg":"path is \"/data/regions/9e71e860-2560-46f3-bfb7-9fc4638fa7ab\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:51.514012795Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"datafile"}
{"msg":"dataset is oxp_784212b9-62b7-432d-b0d7-e4600fa06921/crucible/regions/9e71e860-2560-46f3-bfb7-9fc4638fa7ab","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:51.523318319Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"datafile"}
{"msg":"snapshot is oxp_784212b9-62b7-432d-b0d7-e4600fa06921/crucible/regions/9e71e860-2560-46f3-bfb7-9fc4638fa7ab@728cd9b1-e3ec-4e7d-b786-e0e10f0569d2","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:51.531712431Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_784212b9-62b7-432d-b0d7-e4600fa06921/crucible/regions/9e71e860-2560-46f3-bfb7-9fc4638fa7ab\", \"728cd9b1-e3ec-4e7d-b786-e0e10f0569d2\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:51.531745182Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:51.542559406Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"uri":"/crucible/0/regions/9e71e860-2560-46f3-bfb7-9fc4638fa7ab/snapshots/728cd9b1-e3ec-4e7d-b786-e0e10f0569d2","method":"GET","req_id":"d15382a6-bd8f-4862-ad27-df44e860e576","remote_addr":"[fd00:1122:3344:101::a]:58437","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":28571,"response_code":"200"}
{"msg":"path is \"/data/regions/9e71e860-2560-46f3-bfb7-9fc4638fa7ab\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:51.550452251Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"datafile"}
{"msg":"dataset is oxp_784212b9-62b7-432d-b0d7-e4600fa06921/crucible/regions/9e71e860-2560-46f3-bfb7-9fc4638fa7ab","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:51.559197098Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"datafile"}
{"msg":"snapshot is oxp_784212b9-62b7-432d-b0d7-e4600fa06921/crucible/regions/9e71e860-2560-46f3-bfb7-9fc4638fa7ab@728cd9b1-e3ec-4e7d-b786-e0e10f0569d2","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:51.568418181Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_784212b9-62b7-432d-b0d7-e4600fa06921/crucible/regions/9e71e860-2560-46f3-bfb7-9fc4638fa7ab\", \"728cd9b1-e3ec-4e7d-b786-e0e10f0569d2\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:51.568455062Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 9e71e860-2560-46f3-bfb7-9fc4638fa7ab-728cd9b1-e3ec-4e7d-b786-e0e10f0569d2 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:51.576838554Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"datafile"}
{"msg":"applying SMF actions for region 9e71e860-2560-46f3-bfb7-9fc4638fa7ab running snapshot 728cd9b1-e3ec-4e7d-b786-e0e10f0569d2 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:51.577810216Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:51.577835507Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"uri":"/crucible/0/regions/9e71e860-2560-46f3-bfb7-9fc4638fa7ab/snapshots/728cd9b1-e3ec-4e7d-b786-e0e10f0569d2/run","method":"POST","req_id":"9904af0b-4c33-422a-a738-8d642dfd4412","remote_addr":"[fd00:1122:3344:101::a]:58437","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":27420,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-9e71e860-2560-46f3-bfb7-9fc4638fa7ab-728cd9b1-e3ec-4e7d-b786-e0e10f0569d2","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:51.585211715Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-9e71e860-2560-46f3-bfb7-9fc4638fa7ab-728cd9b1-e3ec-4e7d-b786-e0e10f0569d2","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:51.597003483Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-9e71e860-2560-46f3-bfb7-9fc4638fa7ab-728cd9b1-e3ec-4e7d-b786-e0e10f0569d2","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:51.619903599Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/9e71e860-2560-46f3-bfb7-9fc4638fa7ab/.zfs/snapshot/728cd9b1-e3ec-4e7d-b786-e0e10f0569d2","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:51.620245554Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:51.620309644Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:51.620358665Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:51.620406056Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:51.620467926Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:51.627927846Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:51.636608422Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"region 9e71e860-2560-46f3-bfb7-9fc4638fa7ab running snapshot 728cd9b1-e3ec-4e7d-b786-e0e10f0569d2 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:51.636616242Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"datafile"}
{"msg":"path is \"/data/regions/9e71e860-2560-46f3-bfb7-9fc4638fa7ab\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:51.938190292Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"datafile"}
{"msg":"dataset is oxp_784212b9-62b7-432d-b0d7-e4600fa06921/crucible/regions/9e71e860-2560-46f3-bfb7-9fc4638fa7ab","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:51.947198142Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"datafile"}
{"msg":"snapshot is oxp_784212b9-62b7-432d-b0d7-e4600fa06921/crucible/regions/9e71e860-2560-46f3-bfb7-9fc4638fa7ab@728cd9b1-e3ec-4e7d-b786-e0e10f0569d2","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:51.957970636Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_784212b9-62b7-432d-b0d7-e4600fa06921/crucible/regions/9e71e860-2560-46f3-bfb7-9fc4638fa7ab\", \"728cd9b1-e3ec-4e7d-b786-e0e10f0569d2\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:51.958003696Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:51.966278667Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"uri":"/crucible/0/regions/9e71e860-2560-46f3-bfb7-9fc4638fa7ab/snapshots/728cd9b1-e3ec-4e7d-b786-e0e10f0569d2/run","method":"POST","req_id":"b2a24185-0e62-4a4d-ad4d-73b77e77da48","remote_addr":"[fd00:1122:3344:101::a]:58437","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":28692,"response_code":"200"}
{"msg":"region 0b849c22-953e-4b03-bbe1-29d72afb3872 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:56.442212099Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:56.442436862Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"uri":"/crucible/0/regions","method":"POST","req_id":"c444120d-bfdf-47df-8612-272f1fa7cfd5","remote_addr":"[fd00:1122:3344:101::a]:58437","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":293,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:56.442460053Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_784212b9-62b7-432d-b0d7-e4600fa06921/crucible/regions/0b849c22-953e-4b03-bbe1-29d72afb3872","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:56.452354105Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_784212b9-62b7-432d-b0d7-e4600fa06921/crucible/regions/0b849c22-953e-4b03-bbe1-29d72afb3872","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:56.452382925Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"0b849c22-953e-4b03-bbe1-29d72afb3872\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/0b849c22-953e-4b03-bbe1-29d72afb3872\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:56.507602703Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"region":"0b849c22-953e-4b03-bbe1-29d72afb3872","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"0b849c22-953e-4b03-bbe1-29d72afb3872\" \"--data\" \"/data/regions/0b849c22-953e-4b03-bbe1-29d72afb3872\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:56.507661404Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"region":"0b849c22-953e-4b03-bbe1-29d72afb3872","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:56.58826301Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"region":"0b849c22-953e-4b03-bbe1-29d72afb3872","component":"worker"}
{"msg":"region 0b849c22-953e-4b03-bbe1-29d72afb3872 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:56.588320791Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:56.588444003Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-0b849c22-953e-4b03-bbe1-29d72afb3872","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:56.589629569Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-0b849c22-953e-4b03-bbe1-29d72afb3872","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:56.593625912Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:56.603351012Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-0b849c22-953e-4b03-bbe1-29d72afb3872","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:56.607395026Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/0b849c22-953e-4b03-bbe1-29d72afb3872","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:56.607769761Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:56.607848342Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:56.607912303Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:56.607974614Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:56.612990671Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:56.617804145Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"uri":"/crucible/0/regions","method":"POST","req_id":"9606b7ce-458a-4b56-b87c-bc5194dcba50","remote_addr":"[fd00:1122:3344:101::a]:58437","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":99,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T19:44:56.626772065Z","hostname":"oxz_crucible_5bce18f9-02f8-49f9-ba47-3434651d04e1","pid":23516,"component":"worker"}