[ Jul 23 20:56:59 Enabled. ]
[ Jul 23 20:57:00 Rereading configuration. ]
[ Jul 23 20:57:01 Rereading configuration. ]
[ Jul 23 20:57:03 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_ece1ea9a-7386-4d98-9016-812b5f7f9b23/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_ece1ea9a-7386-4d98-9016-812b5f7f9b23/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:57:03.37338536Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298}
{"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:57:03.375104738Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:57:03.375119198Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:57:03.395428069Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:57:03.493218205Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:58:05.520466085Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:52439"}
{"msg":"region 4faad690-35d6-4f39-b3ec-f527f211d061 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:58:05.539269043Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:58:05.546602499Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298,"uri":"/crucible/0/regions","method":"POST","req_id":"6de10ea8-e3a1-4ecf-a736-db5afb73bc25","remote_addr":"[fd00:1122:3344:101::a]:52439","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":15316,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:58:05.54662456Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_ece1ea9a-7386-4d98-9016-812b5f7f9b23/crucible/regions/4faad690-35d6-4f39-b3ec-f527f211d061","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:58:05.560628352Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_ece1ea9a-7386-4d98-9016-812b5f7f9b23/crucible/regions/4faad690-35d6-4f39-b3ec-f527f211d061","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:58:05.560704653Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"4faad690-35d6-4f39-b3ec-f527f211d061\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/4faad690-35d6-4f39-b3ec-f527f211d061\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:58:05.649354576Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298,"region":"4faad690-35d6-4f39-b3ec-f527f211d061","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"4faad690-35d6-4f39-b3ec-f527f211d061\" \"--data\" \"/data/regions/4faad690-35d6-4f39-b3ec-f527f211d061\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:58:05.65147835Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298,"region":"4faad690-35d6-4f39-b3ec-f527f211d061","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:58:05.783550791Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298,"region":"4faad690-35d6-4f39-b3ec-f527f211d061","component":"worker"}
{"msg":"region 4faad690-35d6-4f39-b3ec-f527f211d061 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:58:05.783646663Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:58:05.783852726Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-4faad690-35d6-4f39-b3ec-f527f211d061","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:58:05.784317283Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-4faad690-35d6-4f39-b3ec-f527f211d061","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:58:05.79104609Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:58:05.807532191Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-4faad690-35d6-4f39-b3ec-f527f211d061","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:58:05.812082663Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4faad690-35d6-4f39-b3ec-f527f211d061","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:58:05.812795754Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:58:05.817129863Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:58:05.817210444Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:58:05.817252695Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:58:05.824645452Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:58:05.834080031Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:58:05.861950413Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298,"uri":"/crucible/0/regions","method":"POST","req_id":"4487aedb-4a6f-411c-98d1-173e9885e476","remote_addr":"[fd00:1122:3344:101::a]:52439","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":82,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:58:06.893484094Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298,"uri":"/crucible/0/regions/4faad690-35d6-4f39-b3ec-f527f211d061","method":"GET","req_id":"ef509956-5468-4fc2-8723-31b35afaaad7","remote_addr":"[fd00:1122:3344:101::a]:52439","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":577,"response_code":"200"}
{"msg":"path is \"/data/regions/4faad690-35d6-4f39-b3ec-f527f211d061\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:58:06.895701079Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298,"component":"datafile"}
{"msg":"dataset is oxp_ece1ea9a-7386-4d98-9016-812b5f7f9b23/crucible/regions/4faad690-35d6-4f39-b3ec-f527f211d061","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:58:06.904921174Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298,"component":"datafile"}
{"msg":"region 4faad690-35d6-4f39-b3ec-f527f211d061 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:58:06.914015918Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:58:06.914171791Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:58:06.914239182Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298,"uri":"/crucible/0/regions/4faad690-35d6-4f39-b3ec-f527f211d061","method":"DELETE","req_id":"c65f872e-e2a4-45cc-a1a2-e2ddd4c80c34","remote_addr":"[fd00:1122:3344:101::a]:52439","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":18596,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-4faad690-35d6-4f39-b3ec-f527f211d061 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:58:06.91538218Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:58:06.916487137Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298,"uri":"/crucible/0/regions/4faad690-35d6-4f39-b3ec-f527f211d061","method":"GET","req_id":"a1be9160-2d1c-4628-b105-1a730c864082","remote_addr":"[fd00:1122:3344:101::a]:52439","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":28,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:58:06.91980576Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_ece1ea9a-7386-4d98-9016-812b5f7f9b23/crucible/regions/4faad690-35d6-4f39-b3ec-f527f211d061\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:58:06.929007196Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298,"region":"4faad690-35d6-4f39-b3ec-f527f211d061","component":"worker"}
{"msg":"region 4faad690-35d6-4f39-b3ec-f527f211d061 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:58:06.98557297Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-23T20:58:06.985997956Z","hostname":"oxz_crucible_e968b470-2aae-4a52-aa79-3348a53c2fb3","pid":21298,"uri":"/crucible/0/regions/4faad690-35d6-4f39-b3ec-f527f211d061","method":"GET","req_id":"ded6697a-7937-4919-b66d-12fa11127af5","remote_addr":"[fd00:1122:3344:101::a]:52439","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":58,"response_code":"200"}