[ Sep  6 23:36:39 Enabled. ]
[ Sep  6 23:36:39 Rereading configuration. ]
[ Sep  6 23:36:41 Rereading configuration. ]
[ Sep  6 23:36:43 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_fd8d5a1e-9723-41c1-aa2b-460a6959d6c7/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_fd8d5a1e-9723-41c1-aa2b-460a6959d6c7/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:36:43.502193689Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346}
{"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:36:43.512091273Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:36:43.512105563Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:36:43.545397735Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:36:43.715012364Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:37:46.613732631Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:36249"}
{"msg":"region 2081b84d-1bdf-494a-9089-74fb454e64dc state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:37:46.651144298Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:37:46.655732971Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"uri":"/crucible/0/regions","method":"POST","req_id":"11208d9e-3627-47a2-b3fd-2b89ce3d6512","remote_addr":"[fd00:1122:3344:101::b]:36249","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":23739,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:37:46.660374343Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_fd8d5a1e-9723-41c1-aa2b-460a6959d6c7/crucible/regions/2081b84d-1bdf-494a-9089-74fb454e64dc","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:37:46.67266183Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_fd8d5a1e-9723-41c1-aa2b-460a6959d6c7/crucible/regions/2081b84d-1bdf-494a-9089-74fb454e64dc","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:37:46.67269509Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"2081b84d-1bdf-494a-9089-74fb454e64dc\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/2081b84d-1bdf-494a-9089-74fb454e64dc\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:37:46.727685786Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"region":"2081b84d-1bdf-494a-9089-74fb454e64dc","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"2081b84d-1bdf-494a-9089-74fb454e64dc\" \"--data\" \"/data/regions/2081b84d-1bdf-494a-9089-74fb454e64dc\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:37:46.729863176Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"region":"2081b84d-1bdf-494a-9089-74fb454e64dc","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:37:46.838163214Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"uri":"/crucible/0/regions","method":"POST","req_id":"d368c8de-41dc-450f-a9c5-0986364e5835","remote_addr":"[fd00:1122:3344:101::b]:36249","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":86,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:37:46.985927327Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"region":"2081b84d-1bdf-494a-9089-74fb454e64dc","component":"worker"}
{"msg":"region 2081b84d-1bdf-494a-9089-74fb454e64dc state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:37:46.985971508Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:37:46.986071929Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-2081b84d-1bdf-494a-9089-74fb454e64dc","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:37:46.986472105Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-2081b84d-1bdf-494a-9089-74fb454e64dc","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:37:46.991522443Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:37:47.007405839Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-2081b84d-1bdf-494a-9089-74fb454e64dc","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:37:47.010069375Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2081b84d-1bdf-494a-9089-74fb454e64dc","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:37:47.020676959Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:37:47.022588185Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:37:47.022671706Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:37:47.022731427Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:37:47.027161607Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:37:47.033577203Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:37:47.100325108Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"uri":"/crucible/0/regions","method":"POST","req_id":"0d4f0003-6dd6-4a87-91c6-efad42c82767","remote_addr":"[fd00:1122:3344:101::b]:36249","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":66,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:37:48.140488802Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"uri":"/crucible/0/regions/2081b84d-1bdf-494a-9089-74fb454e64dc","method":"GET","req_id":"58a642a1-9f62-4229-8777-08ad03305557","remote_addr":"[fd00:1122:3344:101::b]:36249","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":102,"response_code":"200"}
{"msg":"path is \"/data/regions/2081b84d-1bdf-494a-9089-74fb454e64dc\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:37:48.143619234Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"datafile"}
{"msg":"dataset is oxp_fd8d5a1e-9723-41c1-aa2b-460a6959d6c7/crucible/regions/2081b84d-1bdf-494a-9089-74fb454e64dc","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:37:48.153287115Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"datafile"}
{"msg":"region 2081b84d-1bdf-494a-9089-74fb454e64dc state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:37:48.162700643Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:37:48.162857225Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:37:48.162927396Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"uri":"/crucible/0/regions/2081b84d-1bdf-494a-9089-74fb454e64dc","method":"DELETE","req_id":"cc3e7411-5844-413a-a62c-c5b75d37a7af","remote_addr":"[fd00:1122:3344:101::b]:36249","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":19342,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-2081b84d-1bdf-494a-9089-74fb454e64dc (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:37:48.16395651Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:37:48.166627836Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"uri":"/crucible/0/regions/2081b84d-1bdf-494a-9089-74fb454e64dc","method":"GET","req_id":"404cdb0b-a6d3-423e-ae65-a7060d18a37d","remote_addr":"[fd00:1122:3344:101::b]:36249","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":31,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:37:48.168105646Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_fd8d5a1e-9723-41c1-aa2b-460a6959d6c7/crucible/regions/2081b84d-1bdf-494a-9089-74fb454e64dc\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:37:48.177710226Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"region":"2081b84d-1bdf-494a-9089-74fb454e64dc","component":"worker"}
{"msg":"region 2081b84d-1bdf-494a-9089-74fb454e64dc state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:37:48.233149488Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:37:48.276852451Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"uri":"/crucible/0/regions/2081b84d-1bdf-494a-9089-74fb454e64dc","method":"GET","req_id":"11032df2-1af9-4bf0-8cc2-655d7d7ec7d2","remote_addr":"[fd00:1122:3344:101::b]:36249","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":58,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:14.720259346Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:37023"}
{"msg":"region ce855c04-116f-4b06-b0f9-1006ece489ed state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:14.721035816Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:14.721227689Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"uri":"/crucible/0/regions","method":"POST","req_id":"393a34ac-d0fa-44a7-875f-d17135d79426","remote_addr":"[fd00:1122:3344:101::a]:37023","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":902,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:14.721246409Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_fd8d5a1e-9723-41c1-aa2b-460a6959d6c7/crucible/regions/ce855c04-116f-4b06-b0f9-1006ece489ed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:14.730292732Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_fd8d5a1e-9723-41c1-aa2b-460a6959d6c7/crucible/regions/ce855c04-116f-4b06-b0f9-1006ece489ed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:14.730325742Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"ce855c04-116f-4b06-b0f9-1006ece489ed\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/ce855c04-116f-4b06-b0f9-1006ece489ed\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:14.780844577Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"region":"ce855c04-116f-4b06-b0f9-1006ece489ed","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"ce855c04-116f-4b06-b0f9-1006ece489ed\" \"--data\" \"/data/regions/ce855c04-116f-4b06-b0f9-1006ece489ed\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:14.782613651Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"region":"ce855c04-116f-4b06-b0f9-1006ece489ed","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:14.852693341Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"region":"ce855c04-116f-4b06-b0f9-1006ece489ed","component":"worker"}
{"msg":"region ce855c04-116f-4b06-b0f9-1006ece489ed state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:14.852727712Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:14.852818993Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-ce855c04-116f-4b06-b0f9-1006ece489ed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:14.854199312Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-ce855c04-116f-4b06-b0f9-1006ece489ed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:14.857255503Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:14.864429641Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ce855c04-116f-4b06-b0f9-1006ece489ed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:14.869889255Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ce855c04-116f-4b06-b0f9-1006ece489ed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:14.870209729Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:14.87026248Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:14.8703012Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:14.870339781Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:14.874028091Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:14.878280479Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:38:14.996642812Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"uri":"/crucible/0/regions","method":"POST","req_id":"32e32bfd-dcbe-4466-81a7-2e8911b49c2f","remote_addr":"[fd00:1122:3344:101::a]:37023","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":67,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:38.613452105Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:44062"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:38.613585837Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"uri":"/crucible/0/regions/ce855c04-116f-4b06-b0f9-1006ece489ed","method":"GET","req_id":"6155228a-1320-4878-a91a-3be46bc8c93a","remote_addr":"[fd00:1122:3344:101::a]:44062","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":50,"response_code":"200"}
{"msg":"path is \"/data/regions/ce855c04-116f-4b06-b0f9-1006ece489ed\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:38.62046199Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"datafile"}
{"msg":"dataset is oxp_fd8d5a1e-9723-41c1-aa2b-460a6959d6c7/crucible/regions/ce855c04-116f-4b06-b0f9-1006ece489ed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:38.631150075Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"datafile"}
{"msg":"snapshot is oxp_fd8d5a1e-9723-41c1-aa2b-460a6959d6c7/crucible/regions/ce855c04-116f-4b06-b0f9-1006ece489ed@14f123e0-b5bf-4d3d-81ae-5bb566e4e4ed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:38.641033209Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_fd8d5a1e-9723-41c1-aa2b-460a6959d6c7/crucible/regions/ce855c04-116f-4b06-b0f9-1006ece489ed\", \"14f123e0-b5bf-4d3d-81ae-5bb566e4e4ed\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:38.64107869Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:38.650601559Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"uri":"/crucible/0/regions/ce855c04-116f-4b06-b0f9-1006ece489ed/snapshots/14f123e0-b5bf-4d3d-81ae-5bb566e4e4ed","method":"GET","req_id":"23f3cac2-a082-49f4-b263-3779e0593227","remote_addr":"[fd00:1122:3344:101::a]:44062","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":33707,"response_code":"200"}
{"msg":"path is \"/data/regions/ce855c04-116f-4b06-b0f9-1006ece489ed\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:38.654419581Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"datafile"}
{"msg":"dataset is oxp_fd8d5a1e-9723-41c1-aa2b-460a6959d6c7/crucible/regions/ce855c04-116f-4b06-b0f9-1006ece489ed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:38.663035697Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"datafile"}
{"msg":"snapshot is oxp_fd8d5a1e-9723-41c1-aa2b-460a6959d6c7/crucible/regions/ce855c04-116f-4b06-b0f9-1006ece489ed@14f123e0-b5bf-4d3d-81ae-5bb566e4e4ed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:38.670900734Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_fd8d5a1e-9723-41c1-aa2b-460a6959d6c7/crucible/regions/ce855c04-116f-4b06-b0f9-1006ece489ed\", \"14f123e0-b5bf-4d3d-81ae-5bb566e4e4ed\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:38.670926995Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot ce855c04-116f-4b06-b0f9-1006ece489ed-14f123e0-b5bf-4d3d-81ae-5bb566e4e4ed state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:38.68016394Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"datafile"}
{"msg":"applying SMF actions for region ce855c04-116f-4b06-b0f9-1006ece489ed running snapshot 14f123e0-b5bf-4d3d-81ae-5bb566e4e4ed (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:38.680790578Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:38.680834429Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"uri":"/crucible/0/regions/ce855c04-116f-4b06-b0f9-1006ece489ed/snapshots/14f123e0-b5bf-4d3d-81ae-5bb566e4e4ed/run","method":"POST","req_id":"2cc642e1-a67b-4776-badf-d2dd3ce2a9fe","remote_addr":"[fd00:1122:3344:101::a]:44062","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":26940,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-ce855c04-116f-4b06-b0f9-1006ece489ed-14f123e0-b5bf-4d3d-81ae-5bb566e4e4ed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:38.690051094Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-ce855c04-116f-4b06-b0f9-1006ece489ed-14f123e0-b5bf-4d3d-81ae-5bb566e4e4ed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:38.695628969Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-ce855c04-116f-4b06-b0f9-1006ece489ed-14f123e0-b5bf-4d3d-81ae-5bb566e4e4ed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:38.714339273Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ce855c04-116f-4b06-b0f9-1006ece489ed/.zfs/snapshot/14f123e0-b5bf-4d3d-81ae-5bb566e4e4ed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:38.714538496Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:38.714588356Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:38.714626377Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:38.714662108Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:38.714704718Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:38.720588828Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:38.725737798Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"worker"}
{"msg":"region ce855c04-116f-4b06-b0f9-1006ece489ed running snapshot 14f123e0-b5bf-4d3d-81ae-5bb566e4e4ed state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:38.725745418Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"datafile"}
{"msg":"path is \"/data/regions/ce855c04-116f-4b06-b0f9-1006ece489ed\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:38.860088569Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"datafile"}
{"msg":"dataset is oxp_fd8d5a1e-9723-41c1-aa2b-460a6959d6c7/crucible/regions/ce855c04-116f-4b06-b0f9-1006ece489ed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:38.868690716Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"datafile"}
{"msg":"snapshot is oxp_fd8d5a1e-9723-41c1-aa2b-460a6959d6c7/crucible/regions/ce855c04-116f-4b06-b0f9-1006ece489ed@14f123e0-b5bf-4d3d-81ae-5bb566e4e4ed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:38.877402224Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_fd8d5a1e-9723-41c1-aa2b-460a6959d6c7/crucible/regions/ce855c04-116f-4b06-b0f9-1006ece489ed\", \"14f123e0-b5bf-4d3d-81ae-5bb566e4e4ed\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:38.877435024Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-06T23:43:38.885128209Z","hostname":"oxz_crucible_cea6def4-eb13-4381-bb48-7596d86978d4","pid":19346,"uri":"/crucible/0/regions/ce855c04-116f-4b06-b0f9-1006ece489ed/snapshots/14f123e0-b5bf-4d3d-81ae-5bb566e4e4ed/run","method":"POST","req_id":"4f20a8b3-d344-4b5f-b446-bca49e4a2333","remote_addr":"[fd00:1122:3344:101::a]:44062","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":25063,"response_code":"200"}