[ Jun 5 19:50:48 Enabled. ] [ Jun 5 19:50:48 Rereading configuration. ] [ Jun 5 19:50:51 Rereading configuration. ] [ Jun 5 19:50:54 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_df264fcd-b75d-4ee3-9b2e-e281836815f9/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"dataset: \"oxp_df264fcd-b75d-4ee3-9b2e-e281836815f9/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:50:54.307310413Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084} {"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:50:54.311032121Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:50:54.311054621Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:50:54.331931424Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:50:54.489369031Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:15.065249199Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:40439"} {"msg":"region 7d19e8b4-a333-4a22-b987-9b954ded1938 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:15.067916225Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:15.068848337Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"uri":"/crucible/0/regions","method":"POST","req_id":"165d81be-84bc-4b25-b42f-8ac29d3ca474","remote_addr":"[fd00:1122:3344:101::a]:40439","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":1851,"response_code":"200"} {"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:15.068870498Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"zfs set reservation of 1342177280 for oxp_df264fcd-b75d-4ee3-9b2e-e281836815f9/crucible/regions/7d19e8b4-a333-4a22-b987-9b954ded1938","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:15.07786609Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"zfs set quota of 3221225472 for oxp_df264fcd-b75d-4ee3-9b2e-e281836815f9/crucible/regions/7d19e8b4-a333-4a22-b987-9b954ded1938","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:15.077897981Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"7d19e8b4-a333-4a22-b987-9b954ded1938\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/7d19e8b4-a333-4a22-b987-9b954ded1938\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:15.130181925Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"region":"7d19e8b4-a333-4a22-b987-9b954ded1938","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"7d19e8b4-a333-4a22-b987-9b954ded1938\" \"--data\" \"/data/regions/7d19e8b4-a333-4a22-b987-9b954ded1938\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:15.130235346Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"region":"7d19e8b4-a333-4a22-b987-9b954ded1938","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:15.214850994Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"uri":"/crucible/0/regions","method":"POST","req_id":"34de593d-e9a4-4c2d-9f9b-b13d6605342d","remote_addr":"[fd00:1122:3344:101::a]:40439","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":656,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:15.248864947Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"region":"7d19e8b4-a333-4a22-b987-9b954ded1938","component":"worker"} {"msg":"region 7d19e8b4-a333-4a22-b987-9b954ded1938 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:15.248895998Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:15.249035791Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-7d19e8b4-a333-4a22-b987-9b954ded1938","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:15.249547183Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-7d19e8b4-a333-4a22-b987-9b954ded1938","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:15.253839925Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:15.263311918Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-7d19e8b4-a333-4a22-b987-9b954ded1938","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:15.265996382Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/7d19e8b4-a333-4a22-b987-9b954ded1938","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:15.266747129Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:15.266835201Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:15.266883992Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:15.266930764Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:15.270501118Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:15.274751688Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:15.808630572Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"uri":"/crucible/0/regions","method":"POST","req_id":"ef6e980a-d650-4be1-bcb0-2cf74e13d14b","remote_addr":"[fd00:1122:3344:101::a]:40439","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":84,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:16.282429667Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"uri":"/crucible/0/regions/7d19e8b4-a333-4a22-b987-9b954ded1938","method":"GET","req_id":"c12aedbd-1e8e-45e9-a71e-b8b72c831cf1","remote_addr":"[fd00:1122:3344:101::a]:40439","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":682,"response_code":"200"} {"msg":"path is \"/data/regions/7d19e8b4-a333-4a22-b987-9b954ded1938\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:16.285486379Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"datafile"} {"msg":"dataset is oxp_df264fcd-b75d-4ee3-9b2e-e281836815f9/crucible/regions/7d19e8b4-a333-4a22-b987-9b954ded1938","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:16.29486056Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"datafile"} {"msg":"region 7d19e8b4-a333-4a22-b987-9b954ded1938 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:16.303181327Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:16.303381201Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:16.303502384Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"uri":"/crucible/0/regions/7d19e8b4-a333-4a22-b987-9b954ded1938","method":"DELETE","req_id":"aeee6b0e-0030-424c-9a49-b3ea428f77b1","remote_addr":"[fd00:1122:3344:101::a]:40439","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":18061,"response_code":"204"} {"msg":"disabling downstairs instance: downstairs-7d19e8b4-a333-4a22-b987-9b954ded1938 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:16.304539919Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:16.309649729Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:16.312494807Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"uri":"/crucible/0/regions/7d19e8b4-a333-4a22-b987-9b954ded1938","method":"GET","req_id":"3eba40f1-472e-4e28-9c33-f66645f4cd26","remote_addr":"[fd00:1122:3344:101::a]:40439","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":494,"response_code":"200"} {"msg":"deleting zfs dataset \"oxp_df264fcd-b75d-4ee3-9b2e-e281836815f9/crucible/regions/7d19e8b4-a333-4a22-b987-9b954ded1938\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:16.319750237Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"region":"7d19e8b4-a333-4a22-b987-9b954ded1938","component":"worker"} {"msg":"region 7d19e8b4-a333-4a22-b987-9b954ded1938 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:16.371227013Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:16.442172497Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"uri":"/crucible/0/regions/7d19e8b4-a333-4a22-b987-9b954ded1938","method":"GET","req_id":"1cf147e3-63cc-4268-a9ce-e4f99cf11898","remote_addr":"[fd00:1122:3344:101::a]:40439","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":55,"response_code":"200"} {"msg":"region fe07e78c-8741-41b8-83ab-211e9fc8e2ea state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:17.686194266Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:17.686344539Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"uri":"/crucible/0/regions","method":"POST","req_id":"658695e5-1e8e-4475-89a7-2d5749dfeb63","remote_addr":"[fd00:1122:3344:101::a]:40439","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":215,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:17.68638665Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_df264fcd-b75d-4ee3-9b2e-e281836815f9/crucible/regions/fe07e78c-8741-41b8-83ab-211e9fc8e2ea","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:17.696303684Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_df264fcd-b75d-4ee3-9b2e-e281836815f9/crucible/regions/fe07e78c-8741-41b8-83ab-211e9fc8e2ea","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:17.696335835Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"fe07e78c-8741-41b8-83ab-211e9fc8e2ea\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/fe07e78c-8741-41b8-83ab-211e9fc8e2ea\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:17.745204839Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"region":"fe07e78c-8741-41b8-83ab-211e9fc8e2ea","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"fe07e78c-8741-41b8-83ab-211e9fc8e2ea\" \"--data\" \"/data/regions/fe07e78c-8741-41b8-83ab-211e9fc8e2ea\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:17.747063253Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"region":"fe07e78c-8741-41b8-83ab-211e9fc8e2ea","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:17.814665059Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"region":"fe07e78c-8741-41b8-83ab-211e9fc8e2ea","component":"worker"} {"msg":"region fe07e78c-8741-41b8-83ab-211e9fc8e2ea state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:17.81471957Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:17.814802312Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-fe07e78c-8741-41b8-83ab-211e9fc8e2ea","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:17.816251766Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-fe07e78c-8741-41b8-83ab-211e9fc8e2ea","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:17.819062962Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:17.825913634Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-fe07e78c-8741-41b8-83ab-211e9fc8e2ea","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:17.829056309Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/fe07e78c-8741-41b8-83ab-211e9fc8e2ea","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:17.829392496Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:17.829447628Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:17.829491239Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:17.82953751Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:17.833083073Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:17.837706083Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:51:18.04473542Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"uri":"/crucible/0/regions","method":"POST","req_id":"2df96bad-f7bf-4cc2-8898-3d45f1cfc221","remote_addr":"[fd00:1122:3344:101::a]:40439","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":84,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:40.232907723Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:39964"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:40.233110258Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"uri":"/crucible/0/regions/fe07e78c-8741-41b8-83ab-211e9fc8e2ea","method":"GET","req_id":"e0e332a8-3629-465f-a15e-8a63d0ffde4f","remote_addr":"[fd00:1122:3344:101::a]:39964","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":58,"response_code":"200"} {"msg":"path is \"/data/regions/fe07e78c-8741-41b8-83ab-211e9fc8e2ea\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:40.236625411Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"datafile"} {"msg":"dataset is oxp_df264fcd-b75d-4ee3-9b2e-e281836815f9/crucible/regions/fe07e78c-8741-41b8-83ab-211e9fc8e2ea","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:40.246522335Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"datafile"} {"msg":"snapshot is oxp_df264fcd-b75d-4ee3-9b2e-e281836815f9/crucible/regions/fe07e78c-8741-41b8-83ab-211e9fc8e2ea@2758d653-a18c-46dd-89b5-047a8476dd7a","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:40.255322552Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_df264fcd-b75d-4ee3-9b2e-e281836815f9/crucible/regions/fe07e78c-8741-41b8-83ab-211e9fc8e2ea\", \"2758d653-a18c-46dd-89b5-047a8476dd7a\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:40.255358453Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:40.268517704Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"uri":"/crucible/0/regions/fe07e78c-8741-41b8-83ab-211e9fc8e2ea/snapshots/2758d653-a18c-46dd-89b5-047a8476dd7a","method":"GET","req_id":"04be2281-4a53-48c1-9ab5-5b1d71d23641","remote_addr":"[fd00:1122:3344:101::a]:39964","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":35377,"response_code":"200"} {"msg":"path is \"/data/regions/fe07e78c-8741-41b8-83ab-211e9fc8e2ea\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:40.268687198Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"datafile"} {"msg":"dataset is oxp_df264fcd-b75d-4ee3-9b2e-e281836815f9/crucible/regions/fe07e78c-8741-41b8-83ab-211e9fc8e2ea","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:40.277598958Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"datafile"} {"msg":"snapshot is oxp_df264fcd-b75d-4ee3-9b2e-e281836815f9/crucible/regions/fe07e78c-8741-41b8-83ab-211e9fc8e2ea@2758d653-a18c-46dd-89b5-047a8476dd7a","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:40.285958066Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_df264fcd-b75d-4ee3-9b2e-e281836815f9/crucible/regions/fe07e78c-8741-41b8-83ab-211e9fc8e2ea\", \"2758d653-a18c-46dd-89b5-047a8476dd7a\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:40.285993357Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot fe07e78c-8741-41b8-83ab-211e9fc8e2ea-2758d653-a18c-46dd-89b5-047a8476dd7a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:40.294840836Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"datafile"} {"msg":"applying SMF actions for region fe07e78c-8741-41b8-83ab-211e9fc8e2ea running snapshot 2758d653-a18c-46dd-89b5-047a8476dd7a (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:40.295392829Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:40.295429039Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"uri":"/crucible/0/regions/fe07e78c-8741-41b8-83ab-211e9fc8e2ea/snapshots/2758d653-a18c-46dd-89b5-047a8476dd7a/run","method":"POST","req_id":"4f41239d-5641-4f5b-bf77-896dd97242a0","remote_addr":"[fd00:1122:3344:101::a]:39964","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":26778,"response_code":"200"} {"msg":"creating missing snapshot instance snapshot-fe07e78c-8741-41b8-83ab-211e9fc8e2ea-2758d653-a18c-46dd-89b5-047a8476dd7a","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:40.301751649Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-fe07e78c-8741-41b8-83ab-211e9fc8e2ea-2758d653-a18c-46dd-89b5-047a8476dd7a","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:40.308544569Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-fe07e78c-8741-41b8-83ab-211e9fc8e2ea-2758d653-a18c-46dd-89b5-047a8476dd7a","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:40.318470183Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/fe07e78c-8741-41b8-83ab-211e9fc8e2ea/.zfs/snapshot/2758d653-a18c-46dd-89b5-047a8476dd7a","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:40.318718899Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:40.31877674Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:40.318828321Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:40.318872812Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:40.318926354Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:40.322778004Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:40.326566094Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"region fe07e78c-8741-41b8-83ab-211e9fc8e2ea running snapshot 2758d653-a18c-46dd-89b5-047a8476dd7a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:40.326581674Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"datafile"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:43.27495861Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:55094"} {"msg":"region f51dc4f2-186c-4e34-bbe3-90b4a5c65995 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:43.275045802Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:43.275212396Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"uri":"/crucible/0/regions","method":"POST","req_id":"ec383e57-cf08-406a-83fe-068d5dfb78d9","remote_addr":"[fd00:1122:3344:101::a]:55094","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":225,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:43.275236106Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_df264fcd-b75d-4ee3-9b2e-e281836815f9/crucible/regions/f51dc4f2-186c-4e34-bbe3-90b4a5c65995","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:43.285577501Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_df264fcd-b75d-4ee3-9b2e-e281836815f9/crucible/regions/f51dc4f2-186c-4e34-bbe3-90b4a5c65995","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:43.285603691Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"f51dc4f2-186c-4e34-bbe3-90b4a5c65995\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f51dc4f2-186c-4e34-bbe3-90b4a5c65995\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:43.340113718Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"region":"f51dc4f2-186c-4e34-bbe3-90b4a5c65995","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f51dc4f2-186c-4e34-bbe3-90b4a5c65995\" \"--data\" \"/data/regions/f51dc4f2-186c-4e34-bbe3-90b4a5c65995\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:43.34192119Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"region":"f51dc4f2-186c-4e34-bbe3-90b4a5c65995","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:43.409074546Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"region":"f51dc4f2-186c-4e34-bbe3-90b4a5c65995","component":"worker"} {"msg":"region f51dc4f2-186c-4e34-bbe3-90b4a5c65995 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:43.409103606Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:43.409213869Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-f51dc4f2-186c-4e34-bbe3-90b4a5c65995","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:43.410202322Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f51dc4f2-186c-4e34-bbe3-90b4a5c65995","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:43.413603703Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:43.420392723Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f51dc4f2-186c-4e34-bbe3-90b4a5c65995","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:43.423384223Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f51dc4f2-186c-4e34-bbe3-90b4a5c65995","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:43.423613559Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:43.423679821Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:43.423731002Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:43.423784713Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:43.427220194Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:43.436916373Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:56:43.43807368Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"uri":"/crucible/0/regions","method":"POST","req_id":"3c1181cb-3d36-4d47-b361-8bf7371cdb75","remote_addr":"[fd00:1122:3344:101::a]:55094","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":86,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:57:08.520633214Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"uri":"/crucible/0/regions/f51dc4f2-186c-4e34-bbe3-90b4a5c65995","method":"GET","req_id":"3220a3a2-4b69-40b7-ae9e-d4d23ceccdb6","remote_addr":"[fd00:1122:3344:101::a]:55094","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":67,"response_code":"200"} {"msg":"path is \"/data/regions/f51dc4f2-186c-4e34-bbe3-90b4a5c65995\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:57:08.523214075Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"datafile"} {"msg":"dataset is oxp_df264fcd-b75d-4ee3-9b2e-e281836815f9/crucible/regions/f51dc4f2-186c-4e34-bbe3-90b4a5c65995","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:57:08.53314595Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"datafile"} {"msg":"region f51dc4f2-186c-4e34-bbe3-90b4a5c65995 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:57:08.540533704Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:57:08.540723099Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:57:08.54076719Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"uri":"/crucible/0/regions/f51dc4f2-186c-4e34-bbe3-90b4a5c65995","method":"DELETE","req_id":"67ff8319-9ed4-4ff4-86a2-25dd857eac1c","remote_addr":"[fd00:1122:3344:101::a]:55094","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":17583,"response_code":"204"} {"msg":"disabling downstairs instance: downstairs-f51dc4f2-186c-4e34-bbe3-90b4a5c65995 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:57:08.542375548Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:57:08.543965295Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"uri":"/crucible/0/regions/f51dc4f2-186c-4e34-bbe3-90b4a5c65995","method":"GET","req_id":"9304aabb-44f2-4ae4-a122-a67fc0ac4ee4","remote_addr":"[fd00:1122:3344:101::a]:55094","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":25,"response_code":"200"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:57:08.54799814Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_df264fcd-b75d-4ee3-9b2e-e281836815f9/crucible/regions/f51dc4f2-186c-4e34-bbe3-90b4a5c65995\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:57:08.557004793Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"region":"f51dc4f2-186c-4e34-bbe3-90b4a5c65995","component":"worker"} {"msg":"region f51dc4f2-186c-4e34-bbe3-90b4a5c65995 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:57:08.60132876Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-05T19:57:08.606874501Z","hostname":"oxz_crucible_bc50e194-c3e3-4fbb-a14d-26741c36b466","pid":23084,"uri":"/crucible/0/regions/f51dc4f2-186c-4e34-bbe3-90b4a5c65995","method":"GET","req_id":"3da849aa-a477-4e22-94fe-b0da66bcc1cb","remote_addr":"[fd00:1122:3344:101::a]:55094","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":664,"response_code":"200"}