[ Sep 1 21:00:13 Enabled. ] [ Sep 1 21:00:13 Rereading configuration. ] [ Sep 1 21:00:15 Rereading configuration. ] [ Sep 1 21:00:16 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_c65e07fa-2bd7-44d6-b309-46d959ab18fb/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"dataset: \"oxp_c65e07fa-2bd7-44d6-b309-46d959ab18fb/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:00:16.745215354Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185} {"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:00:16.749452191Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:00:16.749465951Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:00:16.768468727Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:00:16.894714458Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:00:58.143237132Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:44278"} {"msg":"region 11bccf05-86ad-4e71-8b64-aadc45991d0e state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:00:58.190425108Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:00:58.192905442Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"uri":"/crucible/0/regions","method":"POST","req_id":"1ee4095b-3bba-46f8-9114-032643723033","remote_addr":"[fd00:1122:3344:101::a]:44278","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":28360,"response_code":"200"} {"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:00:58.193156405Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"zfs set reservation of 1342177280 for oxp_c65e07fa-2bd7-44d6-b309-46d959ab18fb/crucible/regions/11bccf05-86ad-4e71-8b64-aadc45991d0e","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:00:58.212962372Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"zfs set quota of 3221225472 for oxp_c65e07fa-2bd7-44d6-b309-46d959ab18fb/crucible/regions/11bccf05-86ad-4e71-8b64-aadc45991d0e","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:00:58.212993462Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"11bccf05-86ad-4e71-8b64-aadc45991d0e\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/11bccf05-86ad-4e71-8b64-aadc45991d0e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:00:58.374170485Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"region":"11bccf05-86ad-4e71-8b64-aadc45991d0e","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"11bccf05-86ad-4e71-8b64-aadc45991d0e\" \"--data\" \"/data/regions/11bccf05-86ad-4e71-8b64-aadc45991d0e\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:00:58.376637288Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"region":"11bccf05-86ad-4e71-8b64-aadc45991d0e","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:00:58.43464761Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"uri":"/crucible/0/regions","method":"POST","req_id":"efba4864-5784-499d-b0c9-7fe9938f0ddd","remote_addr":"[fd00:1122:3344:101::a]:44278","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":79,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:00:58.748776315Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"uri":"/crucible/0/regions","method":"POST","req_id":"81336430-7b16-4fad-8d64-51908a1cb27a","remote_addr":"[fd00:1122:3344:101::a]:44278","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":573,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:00:59.06745952Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"region":"11bccf05-86ad-4e71-8b64-aadc45991d0e","component":"worker"} {"msg":"region 11bccf05-86ad-4e71-8b64-aadc45991d0e state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:00:59.067504851Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:00:59.067607272Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-11bccf05-86ad-4e71-8b64-aadc45991d0e","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:00:59.068091728Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-11bccf05-86ad-4e71-8b64-aadc45991d0e","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:00:59.093487181Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:00:59.146477155Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-11bccf05-86ad-4e71-8b64-aadc45991d0e","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:00:59.161249404Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/11bccf05-86ad-4e71-8b64-aadc45991d0e","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:00:59.188944577Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:00:59.191248289Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:00:59.191298929Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:00:59.19133763Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:00:59.212064169Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:00:59.227632119Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:00:59.314833805Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"uri":"/crucible/0/regions","method":"POST","req_id":"52f4b8ab-20d3-4dd5-8086-347b10c20df0","remote_addr":"[fd00:1122:3344:101::a]:44278","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":65,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:01:00.443294636Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"uri":"/crucible/0/regions/11bccf05-86ad-4e71-8b64-aadc45991d0e","method":"GET","req_id":"4a3f396b-d5eb-4d90-8f86-8981cb78adec","remote_addr":"[fd00:1122:3344:101::a]:44278","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":71,"response_code":"200"} {"msg":"path is \"/data/regions/11bccf05-86ad-4e71-8b64-aadc45991d0e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:01:00.446829654Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"datafile"} {"msg":"dataset is oxp_c65e07fa-2bd7-44d6-b309-46d959ab18fb/crucible/regions/11bccf05-86ad-4e71-8b64-aadc45991d0e","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:01:00.457498968Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"datafile"} {"msg":"region 11bccf05-86ad-4e71-8b64-aadc45991d0e state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:01:00.466787883Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:01:00.466946105Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:01:00.467008006Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"uri":"/crucible/0/regions/11bccf05-86ad-4e71-8b64-aadc45991d0e","method":"DELETE","req_id":"eddf639d-4675-46ca-a257-560f6a44181a","remote_addr":"[fd00:1122:3344:101::a]:44278","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":20231,"response_code":"204"} {"msg":"disabling downstairs instance: downstairs-11bccf05-86ad-4e71-8b64-aadc45991d0e (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:01:00.468004759Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:01:00.47102493Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"uri":"/crucible/0/regions/11bccf05-86ad-4e71-8b64-aadc45991d0e","method":"GET","req_id":"16494dfd-2a27-444c-9ac5-213a8c84cc18","remote_addr":"[fd00:1122:3344:101::a]:44278","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":25,"response_code":"200"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:01:00.482609796Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_c65e07fa-2bd7-44d6-b309-46d959ab18fb/crucible/regions/11bccf05-86ad-4e71-8b64-aadc45991d0e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:01:00.492027383Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"region":"11bccf05-86ad-4e71-8b64-aadc45991d0e","component":"worker"} {"msg":"region 11bccf05-86ad-4e71-8b64-aadc45991d0e state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:01:00.594220721Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:01:00.62086677Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"uri":"/crucible/0/regions/11bccf05-86ad-4e71-8b64-aadc45991d0e","method":"GET","req_id":"81ee9454-5606-4dab-b218-97651df8dfc2","remote_addr":"[fd00:1122:3344:101::a]:44278","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":78,"response_code":"200"} {"msg":"region 880459b9-9dc0-473d-a303-fa5be0503fb6 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:01:04.269619834Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:01:04.269767166Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"uri":"/crucible/0/regions","method":"POST","req_id":"e7d78418-4e76-411d-b628-080ae4627ce4","remote_addr":"[fd00:1122:3344:101::a]:44278","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":209,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:01:04.269790646Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_c65e07fa-2bd7-44d6-b309-46d959ab18fb/crucible/regions/880459b9-9dc0-473d-a303-fa5be0503fb6","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:01:04.278898259Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_c65e07fa-2bd7-44d6-b309-46d959ab18fb/crucible/regions/880459b9-9dc0-473d-a303-fa5be0503fb6","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:01:04.278930559Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"880459b9-9dc0-473d-a303-fa5be0503fb6\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/880459b9-9dc0-473d-a303-fa5be0503fb6\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:01:04.422551305Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"region":"880459b9-9dc0-473d-a303-fa5be0503fb6","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"880459b9-9dc0-473d-a303-fa5be0503fb6\" \"--data\" \"/data/regions/880459b9-9dc0-473d-a303-fa5be0503fb6\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:01:04.424415901Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"region":"880459b9-9dc0-473d-a303-fa5be0503fb6","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:01:04.533235777Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"uri":"/crucible/0/regions","method":"POST","req_id":"2ed211f7-fd79-41a2-85b2-a4e293a10301","remote_addr":"[fd00:1122:3344:101::a]:44278","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":81,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:01:04.623856829Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"region":"880459b9-9dc0-473d-a303-fa5be0503fb6","component":"worker"} {"msg":"region 880459b9-9dc0-473d-a303-fa5be0503fb6 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:01:04.62389117Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:01:04.624007291Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-880459b9-9dc0-473d-a303-fa5be0503fb6","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:01:04.62538255Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-880459b9-9dc0-473d-a303-fa5be0503fb6","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:01:04.64250161Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:01:04.669623376Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-880459b9-9dc0-473d-a303-fa5be0503fb6","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:01:04.686601745Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/880459b9-9dc0-473d-a303-fa5be0503fb6","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:01:04.68699042Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:01:04.687063691Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:01:04.687118992Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:01:04.687172952Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:01:04.699985015Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:01:04.709828878Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:01:04.869348618Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"uri":"/crucible/0/regions","method":"POST","req_id":"28c6047a-00a4-4dbe-bdfc-cc2220156a26","remote_addr":"[fd00:1122:3344:101::a]:44278","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":822,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:28.348958179Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:53689"} {"msg":"region 064df0a2-753e-4bfd-80ae-d7bf03126280 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:28.349098021Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:28.34976078Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:28.34977259Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"uri":"/crucible/0/regions","method":"POST","req_id":"d474ee78-604d-49ec-b569-371224e397fc","remote_addr":"[fd00:1122:3344:101::a]:53689","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":766,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_c65e07fa-2bd7-44d6-b309-46d959ab18fb/crucible/regions/064df0a2-753e-4bfd-80ae-d7bf03126280","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:28.362094827Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_c65e07fa-2bd7-44d6-b309-46d959ab18fb/crucible/regions/064df0a2-753e-4bfd-80ae-d7bf03126280","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:28.362130297Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"064df0a2-753e-4bfd-80ae-d7bf03126280\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/064df0a2-753e-4bfd-80ae-d7bf03126280\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:28.429340743Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"region":"064df0a2-753e-4bfd-80ae-d7bf03126280","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"064df0a2-753e-4bfd-80ae-d7bf03126280\" \"--data\" \"/data/regions/064df0a2-753e-4bfd-80ae-d7bf03126280\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:28.431306909Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"region":"064df0a2-753e-4bfd-80ae-d7bf03126280","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:28.53373016Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"region":"064df0a2-753e-4bfd-80ae-d7bf03126280","component":"worker"} {"msg":"region 064df0a2-753e-4bfd-80ae-d7bf03126280 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:28.533789731Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:28.533901252Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-064df0a2-753e-4bfd-80ae-d7bf03126280","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:28.534903676Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-064df0a2-753e-4bfd-80ae-d7bf03126280","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:28.548711952Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:28.572026196Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-064df0a2-753e-4bfd-80ae-d7bf03126280","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:28.575499463Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/064df0a2-753e-4bfd-80ae-d7bf03126280","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:28.575815588Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:28.575868688Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:28.575913449Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:28.575951449Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:28.584334482Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:28.592735096Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:28.611237375Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"uri":"/crucible/0/regions","method":"POST","req_id":"0d6fa109-a4f5-4e4b-a4db-64db499606e3","remote_addr":"[fd00:1122:3344:101::a]:53689","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":77,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:29.482313916Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"uri":"/crucible/0/regions/880459b9-9dc0-473d-a303-fa5be0503fb6","method":"GET","req_id":"4291042f-252c-44fc-940a-5d38725c38fe","remote_addr":"[fd00:1122:3344:101::a]:53689","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":52,"response_code":"200"} {"msg":"path is \"/data/regions/880459b9-9dc0-473d-a303-fa5be0503fb6\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:29.488824184Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"datafile"} {"msg":"dataset is oxp_c65e07fa-2bd7-44d6-b309-46d959ab18fb/crucible/regions/880459b9-9dc0-473d-a303-fa5be0503fb6","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:29.498867669Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"datafile"} {"msg":"snapshot is oxp_c65e07fa-2bd7-44d6-b309-46d959ab18fb/crucible/regions/880459b9-9dc0-473d-a303-fa5be0503fb6@8991ec16-81c2-451b-b0f7-793e02a50efc","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:29.506765576Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_c65e07fa-2bd7-44d6-b309-46d959ab18fb/crucible/regions/880459b9-9dc0-473d-a303-fa5be0503fb6\", \"8991ec16-81c2-451b-b0f7-793e02a50efc\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:29.506796446Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:29.514790844Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"uri":"/crucible/0/regions/880459b9-9dc0-473d-a303-fa5be0503fb6/snapshots/8991ec16-81c2-451b-b0f7-793e02a50efc","method":"GET","req_id":"10cc5cba-21bd-457a-a038-cb906332def7","remote_addr":"[fd00:1122:3344:101::a]:53689","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":28995,"response_code":"200"} {"msg":"path is \"/data/regions/880459b9-9dc0-473d-a303-fa5be0503fb6\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:29.517587742Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"datafile"} {"msg":"dataset is oxp_c65e07fa-2bd7-44d6-b309-46d959ab18fb/crucible/regions/880459b9-9dc0-473d-a303-fa5be0503fb6","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:29.525972915Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"datafile"} {"msg":"snapshot is oxp_c65e07fa-2bd7-44d6-b309-46d959ab18fb/crucible/regions/880459b9-9dc0-473d-a303-fa5be0503fb6@8991ec16-81c2-451b-b0f7-793e02a50efc","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:29.534344957Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_c65e07fa-2bd7-44d6-b309-46d959ab18fb/crucible/regions/880459b9-9dc0-473d-a303-fa5be0503fb6\", \"8991ec16-81c2-451b-b0f7-793e02a50efc\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:29.534368588Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot 880459b9-9dc0-473d-a303-fa5be0503fb6-8991ec16-81c2-451b-b0f7-793e02a50efc state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:29.54270372Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"datafile"} {"msg":"applying SMF actions for region 880459b9-9dc0-473d-a303-fa5be0503fb6 running snapshot 8991ec16-81c2-451b-b0f7-793e02a50efc (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:29.542895153Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:29.542934293Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"uri":"/crucible/0/regions/880459b9-9dc0-473d-a303-fa5be0503fb6/snapshots/8991ec16-81c2-451b-b0f7-793e02a50efc/run","method":"POST","req_id":"cab6acc2-d3b1-45f3-b649-570655114ede","remote_addr":"[fd00:1122:3344:101::a]:53689","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":25378,"response_code":"200"} {"msg":"creating missing snapshot instance snapshot-880459b9-9dc0-473d-a303-fa5be0503fb6-8991ec16-81c2-451b-b0f7-793e02a50efc","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:29.54562573Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-880459b9-9dc0-473d-a303-fa5be0503fb6-8991ec16-81c2-451b-b0f7-793e02a50efc","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:29.552086597Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-880459b9-9dc0-473d-a303-fa5be0503fb6-8991ec16-81c2-451b-b0f7-793e02a50efc","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:29.570068739Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/880459b9-9dc0-473d-a303-fa5be0503fb6/.zfs/snapshot/8991ec16-81c2-451b-b0f7-793e02a50efc","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:29.570392083Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:29.570468124Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:29.570518625Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:29.570568026Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:29.570644357Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:29.577173005Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:29.583158486Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"region 880459b9-9dc0-473d-a303-fa5be0503fb6 running snapshot 8991ec16-81c2-451b-b0f7-793e02a50efc state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:29.583171286Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"datafile"} {"msg":"path is \"/data/regions/880459b9-9dc0-473d-a303-fa5be0503fb6\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:29.83792843Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"datafile"} {"msg":"dataset is oxp_c65e07fa-2bd7-44d6-b309-46d959ab18fb/crucible/regions/880459b9-9dc0-473d-a303-fa5be0503fb6","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:29.846205692Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"datafile"} {"msg":"snapshot is oxp_c65e07fa-2bd7-44d6-b309-46d959ab18fb/crucible/regions/880459b9-9dc0-473d-a303-fa5be0503fb6@8991ec16-81c2-451b-b0f7-793e02a50efc","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:29.853155895Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_c65e07fa-2bd7-44d6-b309-46d959ab18fb/crucible/regions/880459b9-9dc0-473d-a303-fa5be0503fb6\", \"8991ec16-81c2-451b-b0f7-793e02a50efc\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:29.853188586Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:29.861100042Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"uri":"/crucible/0/regions/880459b9-9dc0-473d-a303-fa5be0503fb6/snapshots/8991ec16-81c2-451b-b0f7-793e02a50efc/run","method":"POST","req_id":"f1b42f14-f942-40ea-b299-476e6728b5a4","remote_addr":"[fd00:1122:3344:101::a]:53689","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":23231,"response_code":"200"} {"msg":"region cf67034d-7e2d-403b-907b-5a2ec094fd01 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:33.776268137Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:33.77645556Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"uri":"/crucible/0/regions","method":"POST","req_id":"f57960bd-6e66-4268-b8b2-2441e44df645","remote_addr":"[fd00:1122:3344:101::a]:53689","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":251,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:33.77647789Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_c65e07fa-2bd7-44d6-b309-46d959ab18fb/crucible/regions/cf67034d-7e2d-403b-907b-5a2ec094fd01","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:33.786242242Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_c65e07fa-2bd7-44d6-b309-46d959ab18fb/crucible/regions/cf67034d-7e2d-403b-907b-5a2ec094fd01","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:33.786303032Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"cf67034d-7e2d-403b-907b-5a2ec094fd01\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/cf67034d-7e2d-403b-907b-5a2ec094fd01\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:33.848496821Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"region":"cf67034d-7e2d-403b-907b-5a2ec094fd01","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"cf67034d-7e2d-403b-907b-5a2ec094fd01\" \"--data\" \"/data/regions/cf67034d-7e2d-403b-907b-5a2ec094fd01\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:33.850522958Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"region":"cf67034d-7e2d-403b-907b-5a2ec094fd01","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:33.918880999Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"region":"cf67034d-7e2d-403b-907b-5a2ec094fd01","component":"worker"} {"msg":"region cf67034d-7e2d-403b-907b-5a2ec094fd01 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:33.918912319Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:33.919049131Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-cf67034d-7e2d-403b-907b-5a2ec094fd01","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:33.921483594Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-cf67034d-7e2d-403b-907b-5a2ec094fd01","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:33.924441014Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:33.932950839Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-cf67034d-7e2d-403b-907b-5a2ec094fd01","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:33.937283927Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/cf67034d-7e2d-403b-907b-5a2ec094fd01","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:33.93749716Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19003","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:33.937551621Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:33.937603311Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:33.937642312Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:33.942056521Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:33.948234685Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"uri":"/crucible/0/regions","method":"POST","req_id":"81ba05d9-614c-4e45-a11f-559bce15f737","remote_addr":"[fd00:1122:3344:101::a]:53689","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":106,"response_code":"200"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:06:33.948699541Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:07:21.412252597Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"uri":"/crucible/0/regions/cf67034d-7e2d-403b-907b-5a2ec094fd01","method":"GET","req_id":"e6117a45-cabb-4b3e-b110-09ae8d31e2e8","remote_addr":"[fd00:1122:3344:101::a]:53689","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":55,"response_code":"200"} {"msg":"path is \"/data/regions/cf67034d-7e2d-403b-907b-5a2ec094fd01\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:07:21.416642396Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"datafile"} {"msg":"dataset is oxp_c65e07fa-2bd7-44d6-b309-46d959ab18fb/crucible/regions/cf67034d-7e2d-403b-907b-5a2ec094fd01","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:07:21.427849187Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"datafile"} {"msg":"region cf67034d-7e2d-403b-907b-5a2ec094fd01 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:07:21.436763418Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:07:21.436955Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:07:21.437010011Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"uri":"/crucible/0/regions/cf67034d-7e2d-403b-907b-5a2ec094fd01","method":"DELETE","req_id":"26642d5c-6027-4a7c-a1d1-62df09bad257","remote_addr":"[fd00:1122:3344:101::a]:53689","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":20388,"response_code":"204"} {"msg":"disabling downstairs instance: downstairs-cf67034d-7e2d-403b-907b-5a2ec094fd01 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:07:21.438974697Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:07:21.441002995Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"uri":"/crucible/0/regions/cf67034d-7e2d-403b-907b-5a2ec094fd01","method":"GET","req_id":"9224ed53-3275-4bc8-8adf-50ccb242e988","remote_addr":"[fd00:1122:3344:101::a]:53689","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":26,"response_code":"200"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:07:21.445697268Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_c65e07fa-2bd7-44d6-b309-46d959ab18fb/crucible/regions/cf67034d-7e2d-403b-907b-5a2ec094fd01\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:07:21.454925352Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"region":"cf67034d-7e2d-403b-907b-5a2ec094fd01","component":"worker"} {"msg":"region cf67034d-7e2d-403b-907b-5a2ec094fd01 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:07:21.505063088Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-01T21:07:21.581972195Z","hostname":"oxz_crucible_4091dac0-a803-4f41-a822-732fdb74b5db","pid":26185,"uri":"/crucible/0/regions/cf67034d-7e2d-403b-907b-5a2ec094fd01","method":"GET","req_id":"cdcfcbfb-42f9-4e70-a4e4-f78bf090a0a3","remote_addr":"[fd00:1122:3344:101::a]:53689","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":69,"response_code":"200"}