[ Oct 12 03:53:12 Enabled. ]
[ Oct 12 03:53:12 Rereading configuration. ]
[ Oct 12 03:53:13 Rereading configuration. ]
[ Oct 12 03:53:14 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_f95ba139-c738-48a5-85f9-774bb88e7e5a/crucible -l [fd00:1122:3344:101::1a]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_f95ba139-c738-48a5-85f9-774bb88e7e5a/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:14.915178011Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860}
{"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:14.91672693Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:14.916738211Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:14.934694004Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:15.114442875Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.599541806Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:39754"}
{"msg":"region b708e9fd-1757-47fa-b048-9a9743821046 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.609981106Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.610685614Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"uri":"/crucible/0/regions","method":"POST","req_id":"74d3ced1-74ba-4697-a284-59dd40b659e4","remote_addr":"[fd00:1122:3344:101::c]:39754","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":2939,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.614994078Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_f95ba139-c738-48a5-85f9-774bb88e7e5a/crucible/regions/b708e9fd-1757-47fa-b048-9a9743821046","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.624569997Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_f95ba139-c738-48a5-85f9-774bb88e7e5a/crucible/regions/b708e9fd-1757-47fa-b048-9a9743821046","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.624608557Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"b708e9fd-1757-47fa-b048-9a9743821046\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b708e9fd-1757-47fa-b048-9a9743821046\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.675778922Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"region":"b708e9fd-1757-47fa-b048-9a9743821046","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b708e9fd-1757-47fa-b048-9a9743821046\" \"--data\" \"/data/regions/b708e9fd-1757-47fa-b048-9a9743821046\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.678320574Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"region":"b708e9fd-1757-47fa-b048-9a9743821046","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.796080956Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"region":"b708e9fd-1757-47fa-b048-9a9743821046","component":"worker"}
{"msg":"region b708e9fd-1757-47fa-b048-9a9743821046 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.796141966Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.796211817Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-b708e9fd-1757-47fa-b048-9a9743821046","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.796649333Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b708e9fd-1757-47fa-b048-9a9743821046","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.80043867Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.812062544Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b708e9fd-1757-47fa-b048-9a9743821046","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.815029991Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b708e9fd-1757-47fa-b048-9a9743821046","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.815817231Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.815881342Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.815927642Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.815971843Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.819869744Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.823897271Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.870986846Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"uri":"/crucible/0/regions","method":"POST","req_id":"8b42817b-24b5-4dd4-ae10-074b2af55b98","remote_addr":"[fd00:1122:3344:101::c]:39754","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":80,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:40.310347849Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"uri":"/crucible/0/regions/b708e9fd-1757-47fa-b048-9a9743821046","method":"GET","req_id":"4929482d-c4d9-453d-a75d-f6ec29679d4c","remote_addr":"[fd00:1122:3344:101::c]:39754","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":86,"response_code":"200"}
{"msg":"path is \"/data/regions/b708e9fd-1757-47fa-b048-9a9743821046\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:40.314654903Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"datafile"}
{"msg":"dataset is oxp_f95ba139-c738-48a5-85f9-774bb88e7e5a/crucible/regions/b708e9fd-1757-47fa-b048-9a9743821046","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:40.325314935Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"datafile"}
{"msg":"region b708e9fd-1757-47fa-b048-9a9743821046 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:40.334753323Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:40.334902424Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:40.334979435Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"uri":"/crucible/0/regions/b708e9fd-1757-47fa-b048-9a9743821046","method":"DELETE","req_id":"1a0ed9be-e68c-4830-8a03-03edb50954eb","remote_addr":"[fd00:1122:3344:101::c]:39754","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":20356,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-b708e9fd-1757-47fa-b048-9a9743821046 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:40.335969167Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:40.338549479Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"uri":"/crucible/0/regions/b708e9fd-1757-47fa-b048-9a9743821046","method":"GET","req_id":"7aa4e3fa-39c5-4922-9b37-c16799b2000e","remote_addr":"[fd00:1122:3344:101::c]:39754","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":42,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:40.340963149Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_f95ba139-c738-48a5-85f9-774bb88e7e5a/crucible/regions/b708e9fd-1757-47fa-b048-9a9743821046\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:40.352098378Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"region":"b708e9fd-1757-47fa-b048-9a9743821046","component":"worker"}
{"msg":"region b708e9fd-1757-47fa-b048-9a9743821046 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:40.420602898Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:40.449626448Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"uri":"/crucible/0/regions/b708e9fd-1757-47fa-b048-9a9743821046","method":"GET","req_id":"1c7b5d05-7aec-4e53-89fe-53ba499a6b36","remote_addr":"[fd00:1122:3344:101::c]:39754","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":76,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:54:31.286401476Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:63391"}
{"msg":"region 16ebec2e-d65c-4f49-90bc-15e67e9e0c67 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:54:31.287176205Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:54:31.287348498Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:54:31.287363188Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"uri":"/crucible/0/regions","method":"POST","req_id":"0f54ca66-c681-49f8-9ab7-92c16ff5ac4b","remote_addr":"[fd00:1122:3344:101::a]:63391","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":819,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_f95ba139-c738-48a5-85f9-774bb88e7e5a/crucible/regions/16ebec2e-d65c-4f49-90bc-15e67e9e0c67","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:54:31.296329889Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_f95ba139-c738-48a5-85f9-774bb88e7e5a/crucible/regions/16ebec2e-d65c-4f49-90bc-15e67e9e0c67","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:54:31.2963643Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"16ebec2e-d65c-4f49-90bc-15e67e9e0c67\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/16ebec2e-d65c-4f49-90bc-15e67e9e0c67\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:54:31.358982337Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"region":"16ebec2e-d65c-4f49-90bc-15e67e9e0c67","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"16ebec2e-d65c-4f49-90bc-15e67e9e0c67\" \"--data\" \"/data/regions/16ebec2e-d65c-4f49-90bc-15e67e9e0c67\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:54:31.360880731Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"region":"16ebec2e-d65c-4f49-90bc-15e67e9e0c67","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:54:31.42765486Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"region":"16ebec2e-d65c-4f49-90bc-15e67e9e0c67","component":"worker"}
{"msg":"region 16ebec2e-d65c-4f49-90bc-15e67e9e0c67 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:54:31.4276868Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:54:31.427820722Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-16ebec2e-d65c-4f49-90bc-15e67e9e0c67","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:54:31.42926328Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-16ebec2e-d65c-4f49-90bc-15e67e9e0c67","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:54:31.432330318Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:54:31.439603448Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-16ebec2e-d65c-4f49-90bc-15e67e9e0c67","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:54:31.44464367Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/16ebec2e-d65c-4f49-90bc-15e67e9e0c67","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:54:31.444964464Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:54:31.445036185Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:54:31.445080396Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:54:31.445127776Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:54:31.449821495Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:54:31.454858517Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:54:31.52923423Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"uri":"/crucible/0/regions","method":"POST","req_id":"c6345cf8-7b1a-4f64-8a71-d35bf196d242","remote_addr":"[fd00:1122:3344:101::a]:63391","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":62,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:57.552333047Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:42214"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:57.55249186Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"uri":"/crucible/0/regions/16ebec2e-d65c-4f49-90bc-15e67e9e0c67","method":"GET","req_id":"4a7e4f1c-ddd6-423e-94cb-5a063c315898","remote_addr":"[fd00:1122:3344:101::a]:42214","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":60,"response_code":"200"}
{"msg":"path is \"/data/regions/16ebec2e-d65c-4f49-90bc-15e67e9e0c67\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:57.555572988Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"datafile"}
{"msg":"dataset is oxp_f95ba139-c738-48a5-85f9-774bb88e7e5a/crucible/regions/16ebec2e-d65c-4f49-90bc-15e67e9e0c67","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:57.564787502Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"datafile"}
{"msg":"snapshot is oxp_f95ba139-c738-48a5-85f9-774bb88e7e5a/crucible/regions/16ebec2e-d65c-4f49-90bc-15e67e9e0c67@47674216-0197-4c88-8f1b-085c1e8d42a3","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:57.573674632Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_f95ba139-c738-48a5-85f9-774bb88e7e5a/crucible/regions/16ebec2e-d65c-4f49-90bc-15e67e9e0c67\", \"47674216-0197-4c88-8f1b-085c1e8d42a3\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:57.573707323Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:57.587080219Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"uri":"/crucible/0/regions/16ebec2e-d65c-4f49-90bc-15e67e9e0c67/snapshots/47674216-0197-4c88-8f1b-085c1e8d42a3","method":"GET","req_id":"5e730e97-1be8-4bb6-bb43-91eeae13af95","remote_addr":"[fd00:1122:3344:101::a]:42214","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":31539,"response_code":"200"}
{"msg":"path is \"/data/regions/16ebec2e-d65c-4f49-90bc-15e67e9e0c67\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:57.591538224Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"datafile"}
{"msg":"dataset is oxp_f95ba139-c738-48a5-85f9-774bb88e7e5a/crucible/regions/16ebec2e-d65c-4f49-90bc-15e67e9e0c67","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:57.600415735Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"datafile"}
{"msg":"snapshot is oxp_f95ba139-c738-48a5-85f9-774bb88e7e5a/crucible/regions/16ebec2e-d65c-4f49-90bc-15e67e9e0c67@47674216-0197-4c88-8f1b-085c1e8d42a3","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:57.609073222Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_f95ba139-c738-48a5-85f9-774bb88e7e5a/crucible/regions/16ebec2e-d65c-4f49-90bc-15e67e9e0c67\", \"47674216-0197-4c88-8f1b-085c1e8d42a3\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:57.609138843Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 16ebec2e-d65c-4f49-90bc-15e67e9e0c67-47674216-0197-4c88-8f1b-085c1e8d42a3 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:57.617861781Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"datafile"}
{"msg":"applying SMF actions for region 16ebec2e-d65c-4f49-90bc-15e67e9e0c67 running snapshot 47674216-0197-4c88-8f1b-085c1e8d42a3 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:57.618754492Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:57.618768422Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"uri":"/crucible/0/regions/16ebec2e-d65c-4f49-90bc-15e67e9e0c67/snapshots/47674216-0197-4c88-8f1b-085c1e8d42a3/run","method":"POST","req_id":"618c3a9c-1e1b-4ca1-943b-ce8c87d803ac","remote_addr":"[fd00:1122:3344:101::a]:42214","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":27848,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-16ebec2e-d65c-4f49-90bc-15e67e9e0c67-47674216-0197-4c88-8f1b-085c1e8d42a3","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:57.627765064Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-16ebec2e-d65c-4f49-90bc-15e67e9e0c67-47674216-0197-4c88-8f1b-085c1e8d42a3","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:57.632587454Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-16ebec2e-d65c-4f49-90bc-15e67e9e0c67-47674216-0197-4c88-8f1b-085c1e8d42a3","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:57.644039186Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/16ebec2e-d65c-4f49-90bc-15e67e9e0c67/.zfs/snapshot/47674216-0197-4c88-8f1b-085c1e8d42a3","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:57.64435912Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:57.644402261Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:57.644440711Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:57.644480771Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:57.644525922Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:57.648212838Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:57.652287128Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"worker"}
{"msg":"region 16ebec2e-d65c-4f49-90bc-15e67e9e0c67 running snapshot 47674216-0197-4c88-8f1b-085c1e8d42a3 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:57.652299639Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"datafile"}
{"msg":"path is \"/data/regions/16ebec2e-d65c-4f49-90bc-15e67e9e0c67\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:57.850087494Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"datafile"}
{"msg":"dataset is oxp_f95ba139-c738-48a5-85f9-774bb88e7e5a/crucible/regions/16ebec2e-d65c-4f49-90bc-15e67e9e0c67","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:57.858520888Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"datafile"}
{"msg":"snapshot is oxp_f95ba139-c738-48a5-85f9-774bb88e7e5a/crucible/regions/16ebec2e-d65c-4f49-90bc-15e67e9e0c67@47674216-0197-4c88-8f1b-085c1e8d42a3","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:57.866929463Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_f95ba139-c738-48a5-85f9-774bb88e7e5a/crucible/regions/16ebec2e-d65c-4f49-90bc-15e67e9e0c67\", \"47674216-0197-4c88-8f1b-085c1e8d42a3\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:57.866977993Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:57.875265146Z","hostname":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","pid":21860,"uri":"/crucible/0/regions/16ebec2e-d65c-4f49-90bc-15e67e9e0c67/snapshots/47674216-0197-4c88-8f1b-085c1e8d42a3/run","method":"POST","req_id":"df789a37-329d-407a-a597-e9dd1c2e4f6e","remote_addr":"[fd00:1122:3344:101::a]:42214","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":25177,"response_code":"200"}