[ Jul 19 18:45:09 Enabled. ]
[ Jul 19 18:45:09 Rereading configuration. ]
[ Jul 19 18:45:11 Rereading configuration. ]
[ Jul 19 18:45:13 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_ceb75e0c-fc60-474c-85f7-e6b93e4d0425/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_ceb75e0c-fc60-474c-85f7-e6b93e4d0425/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:13.787944744Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028}
{"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:13.791950427Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:13.791963327Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:13.805720021Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:13.940188197Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:33.25895422Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:61900"}
{"msg":"region d5d8e980-8daf-48a5-982c-43a33cda9032 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:33.268671112Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:33.271037759Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"uri":"/crucible/0/regions","method":"POST","req_id":"9dacea65-79d2-4640-857a-2b5600ba9277","remote_addr":"[fd00:1122:3344:101::a]:61900","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":4962,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:33.273413616Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_ceb75e0c-fc60-474c-85f7-e6b93e4d0425/crucible/regions/d5d8e980-8daf-48a5-982c-43a33cda9032","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:33.283737317Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_ceb75e0c-fc60-474c-85f7-e6b93e4d0425/crucible/regions/d5d8e980-8daf-48a5-982c-43a33cda9032","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:33.283806688Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"d5d8e980-8daf-48a5-982c-43a33cda9032\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/d5d8e980-8daf-48a5-982c-43a33cda9032\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:33.35254626Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"region":"d5d8e980-8daf-48a5-982c-43a33cda9032","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"d5d8e980-8daf-48a5-982c-43a33cda9032\" \"--data\" \"/data/regions/d5d8e980-8daf-48a5-982c-43a33cda9032\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:33.354716954Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"region":"d5d8e980-8daf-48a5-982c-43a33cda9032","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:33.477089742Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"region":"d5d8e980-8daf-48a5-982c-43a33cda9032","component":"worker"}
{"msg":"region d5d8e980-8daf-48a5-982c-43a33cda9032 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:33.477155052Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:33.477359456Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-d5d8e980-8daf-48a5-982c-43a33cda9032","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:33.477747932Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-d5d8e980-8daf-48a5-982c-43a33cda9032","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:33.483718985Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:33.497555741Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-d5d8e980-8daf-48a5-982c-43a33cda9032","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:33.501099456Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/d5d8e980-8daf-48a5-982c-43a33cda9032","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:33.501817537Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:33.501912678Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:33.501958359Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:33.50200306Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:33.507058539Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:33.519017045Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:33.555182949Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"uri":"/crucible/0/regions","method":"POST","req_id":"a70eeb95-9bc6-4033-ab43-3571ad0085d2","remote_addr":"[fd00:1122:3344:101::a]:61900","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":81,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:34.086137587Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"uri":"/crucible/0/regions/d5d8e980-8daf-48a5-982c-43a33cda9032","method":"GET","req_id":"acc3dc39-7718-490c-9050-7e777fb9e601","remote_addr":"[fd00:1122:3344:101::a]:61900","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":86,"response_code":"200"}
{"msg":"path is \"/data/regions/d5d8e980-8daf-48a5-982c-43a33cda9032\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:34.088713907Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"datafile"}
{"msg":"dataset is oxp_ceb75e0c-fc60-474c-85f7-e6b93e4d0425/crucible/regions/d5d8e980-8daf-48a5-982c-43a33cda9032","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:34.098417398Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"datafile"}
{"msg":"region d5d8e980-8daf-48a5-982c-43a33cda9032 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:34.106664957Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:34.106825939Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:34.106912601Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"uri":"/crucible/0/regions/d5d8e980-8daf-48a5-982c-43a33cda9032","method":"DELETE","req_id":"f6921592-b878-4838-98a8-c54d304516a1","remote_addr":"[fd00:1122:3344:101::a]:61900","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":18240,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-d5d8e980-8daf-48a5-982c-43a33cda9032 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:34.107996518Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:34.109890827Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"uri":"/crucible/0/regions/d5d8e980-8daf-48a5-982c-43a33cda9032","method":"GET","req_id":"d5bf72ed-85d9-4ada-b43e-93166b791727","remote_addr":"[fd00:1122:3344:101::a]:61900","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":585,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:34.112271404Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_ceb75e0c-fc60-474c-85f7-e6b93e4d0425/crucible/regions/d5d8e980-8daf-48a5-982c-43a33cda9032\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:34.121168813Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"region":"d5d8e980-8daf-48a5-982c-43a33cda9032","component":"worker"}
{"msg":"region d5d8e980-8daf-48a5-982c-43a33cda9032 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:34.169682529Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:34.192898861Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"uri":"/crucible/0/regions/d5d8e980-8daf-48a5-982c-43a33cda9032","method":"GET","req_id":"bbb16968-1211-4960-b89a-8320e5939d38","remote_addr":"[fd00:1122:3344:101::a]:61900","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":62,"response_code":"200"}
{"msg":"region cf24c0a1-091b-41da-952f-aaddb5237975 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:35.82218874Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:35.824232025Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:35.824245346Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"uri":"/crucible/0/regions","method":"POST","req_id":"b799e712-aaf9-42c1-abc6-76087c59bbe0","remote_addr":"[fd00:1122:3344:101::a]:61900","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":2097,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_ceb75e0c-fc60-474c-85f7-e6b93e4d0425/crucible/regions/cf24c0a1-091b-41da-952f-aaddb5237975","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:35.832645817Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_ceb75e0c-fc60-474c-85f7-e6b93e4d0425/crucible/regions/cf24c0a1-091b-41da-952f-aaddb5237975","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:35.832672257Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"cf24c0a1-091b-41da-952f-aaddb5237975\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/cf24c0a1-091b-41da-952f-aaddb5237975\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:35.888223482Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"region":"cf24c0a1-091b-41da-952f-aaddb5237975","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"cf24c0a1-091b-41da-952f-aaddb5237975\" \"--data\" \"/data/regions/cf24c0a1-091b-41da-952f-aaddb5237975\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:35.888271783Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"region":"cf24c0a1-091b-41da-952f-aaddb5237975","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:35.974938165Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"region":"cf24c0a1-091b-41da-952f-aaddb5237975","component":"worker"}
{"msg":"region cf24c0a1-091b-41da-952f-aaddb5237975 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:35.975020736Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:35.975173518Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-cf24c0a1-091b-41da-952f-aaddb5237975","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:35.97653821Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-cf24c0a1-091b-41da-952f-aaddb5237975","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:35.980934358Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:35.989135276Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-cf24c0a1-091b-41da-952f-aaddb5237975","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:35.996099535Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/cf24c0a1-091b-41da-952f-aaddb5237975","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:35.996317468Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:35.996374449Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:35.996418649Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:35.99645204Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:36.000634015Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:36.005886817Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:45:36.068891119Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"uri":"/crucible/0/regions","method":"POST","req_id":"627c4399-e454-44b8-9896-a95efd0f8630","remote_addr":"[fd00:1122:3344:101::a]:61900","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":74,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:51.195379329Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:45288"}
{"msg":"region caf83295-72e6-4960-a6c0-9794bdbd24ca state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:51.195526611Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:51.196362074Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:51.196389305Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"uri":"/crucible/0/regions","method":"POST","req_id":"b82aa496-9271-4120-a293-f612a8ccb22f","remote_addr":"[fd00:1122:3344:101::a]:45288","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":957,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_ceb75e0c-fc60-474c-85f7-e6b93e4d0425/crucible/regions/caf83295-72e6-4960-a6c0-9794bdbd24ca","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:51.206300069Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_ceb75e0c-fc60-474c-85f7-e6b93e4d0425/crucible/regions/caf83295-72e6-4960-a6c0-9794bdbd24ca","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:51.20632359Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"caf83295-72e6-4960-a6c0-9794bdbd24ca\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/caf83295-72e6-4960-a6c0-9794bdbd24ca\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:51.261368808Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"region":"caf83295-72e6-4960-a6c0-9794bdbd24ca","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"caf83295-72e6-4960-a6c0-9794bdbd24ca\" \"--data\" \"/data/regions/caf83295-72e6-4960-a6c0-9794bdbd24ca\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:51.261428519Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"region":"caf83295-72e6-4960-a6c0-9794bdbd24ca","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:51.339653419Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"region":"caf83295-72e6-4960-a6c0-9794bdbd24ca","component":"worker"}
{"msg":"region caf83295-72e6-4960-a6c0-9794bdbd24ca state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:51.339695989Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:51.339795001Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-caf83295-72e6-4960-a6c0-9794bdbd24ca","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:51.340751716Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-caf83295-72e6-4960-a6c0-9794bdbd24ca","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:51.349172477Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:51.363672963Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-caf83295-72e6-4960-a6c0-9794bdbd24ca","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:51.366599349Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/caf83295-72e6-4960-a6c0-9794bdbd24ca","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:51.366823352Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:51.366887533Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:51.366939344Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:51.366989645Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:51.370759614Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:51.376195138Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:51.379401408Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"uri":"/crucible/0/regions","method":"POST","req_id":"3dc8bd93-08a5-4987-85c8-d33fd3f20388","remote_addr":"[fd00:1122:3344:101::a]:45288","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":76,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:52.667814587Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:63244"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:52.667902458Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"uri":"/crucible/0/regions/cf24c0a1-091b-41da-952f-aaddb5237975","method":"GET","req_id":"39e743df-febc-45c1-af01-3eb0162671dd","remote_addr":"[fd00:1122:3344:101::a]:63244","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":48,"response_code":"200"}
{"msg":"path is \"/data/regions/cf24c0a1-091b-41da-952f-aaddb5237975\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:52.668125822Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"datafile"}
{"msg":"dataset is oxp_ceb75e0c-fc60-474c-85f7-e6b93e4d0425/crucible/regions/cf24c0a1-091b-41da-952f-aaddb5237975","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:52.677294455Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"datafile"}
{"msg":"snapshot is oxp_ceb75e0c-fc60-474c-85f7-e6b93e4d0425/crucible/regions/cf24c0a1-091b-41da-952f-aaddb5237975@a465ea46-8db8-45a4-b078-b65a8c9cfbf3","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:52.686052641Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_ceb75e0c-fc60-474c-85f7-e6b93e4d0425/crucible/regions/cf24c0a1-091b-41da-952f-aaddb5237975\", \"a465ea46-8db8-45a4-b078-b65a8c9cfbf3\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:52.686097402Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:52.696036397Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"uri":"/crucible/0/regions/cf24c0a1-091b-41da-952f-aaddb5237975/snapshots/a465ea46-8db8-45a4-b078-b65a8c9cfbf3","method":"GET","req_id":"c1425553-420e-4126-8e8c-1c64864c252b","remote_addr":"[fd00:1122:3344:101::a]:63244","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":27961,"response_code":"200"}
{"msg":"path is \"/data/regions/cf24c0a1-091b-41da-952f-aaddb5237975\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:52.69686139Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"datafile"}
{"msg":"dataset is oxp_ceb75e0c-fc60-474c-85f7-e6b93e4d0425/crucible/regions/cf24c0a1-091b-41da-952f-aaddb5237975","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:52.70517673Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"datafile"}
{"msg":"snapshot is oxp_ceb75e0c-fc60-474c-85f7-e6b93e4d0425/crucible/regions/cf24c0a1-091b-41da-952f-aaddb5237975@a465ea46-8db8-45a4-b078-b65a8c9cfbf3","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:52.713399018Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_ceb75e0c-fc60-474c-85f7-e6b93e4d0425/crucible/regions/cf24c0a1-091b-41da-952f-aaddb5237975\", \"a465ea46-8db8-45a4-b078-b65a8c9cfbf3\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:52.713435628Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot cf24c0a1-091b-41da-952f-aaddb5237975-a465ea46-8db8-45a4-b078-b65a8c9cfbf3 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:52.726061425Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"datafile"}
{"msg":"applying SMF actions for region cf24c0a1-091b-41da-952f-aaddb5237975 running snapshot a465ea46-8db8-45a4-b078-b65a8c9cfbf3 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:52.726224648Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:52.726278239Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"uri":"/crucible/0/regions/cf24c0a1-091b-41da-952f-aaddb5237975/snapshots/a465ea46-8db8-45a4-b078-b65a8c9cfbf3/run","method":"POST","req_id":"462aed1c-c1e6-434b-9c49-79457ec2d3d8","remote_addr":"[fd00:1122:3344:101::a]:63244","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":30088,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-cf24c0a1-091b-41da-952f-aaddb5237975-a465ea46-8db8-45a4-b078-b65a8c9cfbf3","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:52.729052082Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-cf24c0a1-091b-41da-952f-aaddb5237975-a465ea46-8db8-45a4-b078-b65a8c9cfbf3","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:52.734515077Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-cf24c0a1-091b-41da-952f-aaddb5237975-a465ea46-8db8-45a4-b078-b65a8c9cfbf3","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:52.745946575Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/cf24c0a1-091b-41da-952f-aaddb5237975/.zfs/snapshot/a465ea46-8db8-45a4-b078-b65a8c9cfbf3","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:52.746169819Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:52.74622455Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:52.74626533Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:52.746310291Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:52.746361102Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:52.750036549Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:52.75392792Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"worker"}
{"msg":"region cf24c0a1-091b-41da-952f-aaddb5237975 running snapshot a465ea46-8db8-45a4-b078-b65a8c9cfbf3 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T18:50:52.75393827Z","hostname":"oxz_crucible_7c60b212-55fd-4f18-a7f0-5fc2accc5f4d","pid":28028,"component":"datafile"}