@@ -41,4 +41,8 @@ public interface Accountable {
     return Collections.emptyList();
   }

+  /**
+   * An accountable that always returns 0
+   */
+  Accountable NULL_ACCOUNTABLE = () -> 0;
 }

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.lucene.util;

import java.util.Collection;
import java.util.Collections;

/**
 * An object whose RAM usage can be computed.
 *
 * @lucene.internal
 */
public interface Accountable {

  /**
   * Return the memory usage of this object in bytes. Negative values are illegal.
   */
  long ramBytesUsed();

  /**
   * Returns nested resources of this class.
   * The result should be a point-in-time snapshot (to avoid race conditions).
   * @see Accountables
   */
  default Collection<Accountable> getChildResources() {
    return Collections.emptyList();
  }

}

Interface members are always `static final` right?


@@ -72,12 +72,29 @@ public class AppsController {
         });

         this.view.setOnSelectScript(
-                scriptDTO -> scriptInterpreter.runScript(scriptDTO.getScript(), e -> Platform.runLater(() -> {
-                    // no exception if installation is cancelled
-                    if (!(e.getCause() instanceof InterruptedException)) {
-                        new ErrorMessage(tr("The script ended unexpectedly"), e, this.view);
-                    }
-                })));
+                scriptDTO -> {
+                    final StringBuilder environmentBuilder = new StringBuilder();
+                    environmentBuilder.append("TYPE_ID=\"");
+                    environmentBuilder.append(scriptDTO.getTypeId());
+                    environmentBuilder.append("\";\n");
+                    environmentBuilder.append("CATEGORY_ID=\"");
+                    environmentBuilder.append(scriptDTO.getCategoryId());
+                    environmentBuilder.append("\";\n");
+                    environmentBuilder.append("APPLICATION_ID=\"");
+                    environmentBuilder.append(scriptDTO.getApplicationId());
+                    environmentBuilder.append("\";\n");
+                    environmentBuilder.append("SCRIPT_ID=\"");
+                    environmentBuilder.append(scriptDTO.getId());
+                    environmentBuilder.append("\";\n");
+                    final String environment = environmentBuilder.toString();
+                    final String execute = environment + scriptDTO.getScript();
+                    scriptInterpreter.runScript(execute, e -> Platform.runLater(() -> {
+                        // no exception if installation is cancelled
+                        if (!(e.getCause() instanceof InterruptedException)) {
+                            new ErrorMessage(tr("The script ended unexpectedly"), e, this.view);
+                        }
+                    }));
+                });

         onAppLoaded.run();
     }

/*
 * Copyright (C) 2015-2017 PÂRIS Quentin
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */

package org.phoenicis.javafx.controller.apps;

import javafx.application.Platform;
import org.phoenicis.javafx.views.common.ErrorMessage;
import org.phoenicis.javafx.views.common.ThemeManager;
import org.phoenicis.javafx.views.mainwindow.apps.ApplicationsView;
import org.phoenicis.repository.RepositoryManager;
import org.phoenicis.repository.dto.CategoryDTO;
import org.phoenicis.repository.dto.RepositoryDTO;
import org.phoenicis.scripts.interpreter.ScriptInterpreter;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Optional;

import static org.phoenicis.configuration.localisation.Localisation.tr;

public class AppsController {
    private final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(AppsController.class);
    private final ApplicationsView view;
    private final RepositoryManager repositoryManager;
    private final ScriptInterpreter scriptInterpreter;
    private ThemeManager themeManager;

    private Runnable onAppLoaded = () -> {
    };

    public AppsController(ApplicationsView view, RepositoryManager repositoryManager,
            ScriptInterpreter scriptInterpreter,
            ThemeManager themeManager) {
        this.view = view;
        this.repositoryManager = repositoryManager;
        this.scriptInterpreter = scriptInterpreter;
        this.themeManager = themeManager;

        this.repositoryManager.addCallbacks(this::populateView,
                e -> Platform.runLater(() -> view.showFailure(
                        tr("Connecting to the repository failed.\nPlease check your connection and try again."),
                        Optional.of(e))));
    }

    public void loadApps() {
        this.view.showWait();
        this.repositoryManager.triggerRepositoryChange();

        this.view.setOnRetryButtonClicked(event -> {
            this.view.showWait();
            this.repositoryManager.triggerRepositoryChange();
        });

        this.view.setOnSelectScript(
                scriptDTO -> scriptInterpreter.runScript(scriptDTO.getScript(), e -> Platform.runLater(() -> {
                    // no exception if installation is cancelled
                    if (!(e.getCause() instanceof InterruptedException)) {
                        new ErrorMessage(tr("The script ended unexpectedly"), e, this.view);
                    }
                })));

        onAppLoaded.run();
    }

    public void setOnAppLoaded(Runnable onAppLoaded) {
        this.onAppLoaded = onAppLoaded;
    }

    public ApplicationsView getView() {
        return view;
    }

    private void populateView(RepositoryDTO repositoryDTO) {
        Platform.runLater(() -> {
            List<CategoryDTO> categoryDTOS = repositoryDTO.getTypes().get(0).getCategories();
            setDefaultCategoryIcons(categoryDTOS);
            this.view.populate(categoryDTOS);
        });
    }

    private void setDefaultCategoryIcons(List<CategoryDTO> categoryDTOS) {
        try {
            StringBuilder cssBuilder = new StringBuilder();
            for (CategoryDTO category : categoryDTOS) {
                cssBuilder.append("#" + category.getId().toLowerCase() + "Button{\n");
                URI categoryIcon = category.getIcon();
                if (categoryIcon == null) {
                    cssBuilder
                            .append("-fx-background-image: url('/org/phoenicis/javafx/views/common/phoenicis.png');\n");
                } else {
                    cssBuilder.append("-fx-background-image: url('" + categoryIcon + "');\n");
                }
                cssBuilder.append("}\n");
            }
            String css = cssBuilder.toString();
            Path temp = Files.createTempFile("defaultCategoryIcons", ".css").toAbsolutePath();
            File tempFile = temp.toFile();
            tempFile.deleteOnExit();
            Files.write(temp, css.getBytes());
            String defaultCategoryIconsCss = temp.toUri().toString();
            themeManager.setDefaultCategoryIconsCss(defaultCategoryIconsCss);
        } catch (IOException e) {
            LOGGER.warn("Could not set default category icons.", e);
        }
    }
}


@@ -102,7 +102,7 @@ class ExternalDriverSupplier implements Supplier<WebDriver> {
     Optional<Class<? extends Supplier<WebDriver>>> supplierClass = getDelegateClass();
     if (supplierClass.isPresent()) {
       Class<? extends Supplier<WebDriver>> clazz = supplierClass.get();
-      logger.info("Using delegate supplier: " + clazz.getName());
+      logger.finest("Using delegate supplier: " + clazz.getName());
       try {
         @SuppressWarnings("unchecked")
         Constructor<Supplier<WebDriver>> ctor =

// Licensed to the Software Freedom Conservancy (SFC) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The SFC licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

package org.openqa.selenium.testing.drivers;

import static java.util.concurrent.TimeUnit.SECONDS;

import com.google.common.base.Suppliers;

import org.openqa.selenium.Capabilities;
import org.openqa.selenium.ImmutableCapabilities;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.net.UrlChecker;
import org.openqa.selenium.remote.LocalFileDetector;
import org.openqa.selenium.remote.RemoteWebDriver;

import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Optional;
import java.util.function.Supplier;
import java.util.logging.Logger;

/**
 * Supports providing WebDriver instances from an external source using the following system
 * properties:
 *

 *
selenium.external.serverUrl

 *
Defines the fully qualified URL of an external WebDriver server to send commands to.
 *       This server must be compliant with the
 *       JSON wire protocol.
 *       If only this property is provided, then this supplier will provide a new
 *       {@link RemoteWebDriver} instance pointed at the designated server. Otherwise, if a
 *       custom supplier is also defined (see below), this supplier will wait for the server to
 *       be accepting commands before delegating to the designated class for the actual client
 *       creation.
 *

 *
selenium.external.supplierClass

 *
Specifies the fully qualified name of another class on the classpath. This class must
 *       implement {@code Supplier<WebDriver>} and have a public constructor that accepts two
 *       {@link Capabilities} objects as arguments (for the desired and required capabilities,
 *       respectively).
 *

 *

 */
class ExternalDriverSupplier implements Supplier<WebDriver> {
  private static final Logger logger = Logger.getLogger(ExternalDriverSupplier.class.getName());

  private static final String DELEGATE_SUPPLIER_CLASS_PROPERTY = "selenium.external.supplierClass";
  private static final String EXTERNAL_SERVER_URL_PROPERTY = "selenium.external.serverUrl";

  private final Capabilities desiredCapabilities;

  ExternalDriverSupplier(Capabilities desiredCapabilities) {
    this.desiredCapabilities = new ImmutableCapabilities(desiredCapabilities);
  }

  @Override
  public WebDriver get() {
    Optional<Supplier<WebDriver>> delegate = createDelegate(desiredCapabilities);
    delegate = createForExternalServer(desiredCapabilities, delegate);

    return delegate.orElse(Suppliers.ofInstance(null)).get();
  }

  private static Optional<Supplier<WebDriver>> createForExternalServer(
      Capabilities desiredCapabilities,
      Optional<Supplier<WebDriver>> delegate) {
    String externalUrl = System.getProperty(EXTERNAL_SERVER_URL_PROPERTY);
    if (externalUrl != null) {
      logger.info("Using external WebDriver server: " + externalUrl);
      URL url;
      try {
        url = new URL(externalUrl);
      } catch (MalformedURLException e) {
        throw new RuntimeException("Invalid server URL: " + externalUrl, e);
      }
      Supplier<WebDriver> defaultSupplier = new DefaultRemoteSupplier(url, desiredCapabilities);
      Supplier<WebDriver> supplier = new ExternalServerDriverSupplier(
          url, delegate.orElse(defaultSupplier));
      return Optional.of(supplier);
    }
    return delegate;
  }

  private static Optional<Supplier<WebDriver>> createDelegate(Capabilities desiredCapabilities) {
    Optional<Class<? extends Supplier<WebDriver>>> supplierClass = getDelegateClass();
    if (supplierClass.isPresent()) {
      Class<? extends Supplier<WebDriver>> clazz = supplierClass.get();
      logger.info("Using delegate supplier: " + clazz.getName());
      try {
        @SuppressWarnings("unchecked")
        Constructor<Supplier<WebDriver>> ctor =
            (Constructor<Supplier<WebDriver>>) clazz.getConstructor(Capabilities.class);
        return Optional.of(ctor.newInstance(desiredCapabilities));
      } catch (InvocationTargetException e) {
        throw new RuntimeException(e.getTargetException());
      } catch (Exception e) {
        throw new RuntimeException(e);
      }
    }
    return Optional.empty();
  }

  @SuppressWarnings("unchecked")
  private static Optional<Class<? extends Supplier<WebDriver>>> getDelegateClass() {
    String delegateClassName = System.getProperty(DELEGATE_SUPPLIER_CLASS_PROPERTY);
    if (delegateClassName != null) {
      try {
        logger.info("Loading custom supplier: " + delegateClassName);
        Class<? extends Supplier<WebDriver>> clazz =
            (Class<? extends Supplier<WebDriver>>) Class.forName(delegateClassName);
        return Optional.of(clazz);
      } catch (Exception e) {
        throw new RuntimeException(e);
      }
    }
    return Optional.empty();
  }

  /**
   * Waits for an external WebDriver server to be ready before delegating to another supplier
   * for driver creation.
   */
  private static class ExternalServerDriverSupplier implements Supplier<WebDriver> {

    private final URL serverUrl;
    private final Supplier<WebDriver> delegateSupplier;

    private ExternalServerDriverSupplier(
        URL serverUrl, Supplier<WebDriver> delegateSupplier) {
      this.serverUrl = serverUrl;
      this.delegateSupplier = delegateSupplier;
    }

    @Override
    public WebDriver get() {
      try {
        logger.info("Waiting for server to be ready at " + serverUrl);
        new UrlChecker().waitUntilAvailable(60, SECONDS, new URL(serverUrl + "/status"));
        logger.info("Server is ready");
      } catch (UrlChecker.TimeoutException e) {
        throw new RuntimeException("The external server is not accepting commands", e);
      } catch (MalformedURLException e) {
        throw new RuntimeException(e);
      }
      return delegateSupplier.get();
    }
  }

  /**
   * Creates basic {@link RemoteWebDriver} instances.
   */
  private static class DefaultRemoteSupplier implements Supplier<WebDriver> {
    private final URL url;
    private final Capabilities desiredCapabilities;

    private DefaultRemoteSupplier(URL url, Capabilities desiredCapabilities) {
      this.url = url;
      this.desiredCapabilities = desiredCapabilities;
    }

    @Override
    public WebDriver get() {
      RemoteWebDriver driver = new RemoteWebDriver(url, desiredCapabilities);
      driver.setFileDetector(new LocalFileDetector());
      return driver;
    }
  }
}

We chose `info` in the test code for obvious reasons. Changing to `finest` makes debugging harder and noisier.


@@ -54,8 +54,8 @@ public class SDKInfoPluginTest extends InstrumentationTestCase {
 		Context ctx = getInstrumentation().getTargetContext();
 		JSONObject sdkInfo = SDKInfoPlugin.getSDKInfo(ctx);
 		BootConfig bootconfig = BootConfig.getBootConfig(ctx);
-		assertEquals("Wrong app name", "SalesforceSDKTest", sdkInfo.getString("appName"));
-		assertEquals("Wrong app version", "1.0", sdkInfo.getString("appVersion"));
+		assertEquals("Wrong app name", "", sdkInfo.getString("appName"));
+		assertEquals("Wrong app version", "", sdkInfo.getString("appVersion"));
 		List<String> sdkInfoPlugins = toList(sdkInfo.getJSONArray("forcePluginsAvailable"));
 		assertEquals("Wrong number of plugins", 3, sdkInfoPlugins.size());
 		assertTrue("oauth plugin should have been returned", sdkInfoPlugins.contains("com.salesforce.oauth"));

/*
 * Copyright (c) 2012, salesforce.com, inc.
 * All rights reserved.
 * Redistribution and use of this software in source and binary forms, with or
 * without modification, are permitted provided that the following conditions
 * are met:
 * - Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 * - Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 * - Neither the name of salesforce.com, inc. nor the names of its contributors
 * may be used to endorse or promote products derived from this software without
 * specific prior written permission of salesforce.com, inc.
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */
package com.salesforce.androidsdk.phonegap;

import java.util.ArrayList;
import java.util.List;

import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;

import com.salesforce.androidsdk.app.SalesforceSDKManager;
import com.salesforce.androidsdk.rest.BootConfig;

import android.content.Context;
import android.content.pm.PackageManager.NameNotFoundException;
import android.test.InstrumentationTestCase;

/**
 * Tests for SDKInfoPlugin
 *
 */
public class SDKInfoPluginTest extends InstrumentationTestCase {

	/**
	 * Test for getSDKInfo
	 */
	public void testGetSDKInfo() throws NameNotFoundException, JSONException {
		Context ctx = getInstrumentation().getTargetContext();
		JSONObject sdkInfo = SDKInfoPlugin.getSDKInfo(ctx);
		BootConfig bootconfig = BootConfig.getBootConfig(ctx);
		assertEquals("Wrong app name", "SalesforceSDKTest", sdkInfo.getString("appName"));
		assertEquals("Wrong app version", "1.0", sdkInfo.getString("appVersion"));
		List<String> sdkInfoPlugins = toList(sdkInfo.getJSONArray("forcePluginsAvailable"));
		assertEquals("Wrong number of plugins", 3, sdkInfoPlugins.size());
		assertTrue("oauth plugin should have been returned", sdkInfoPlugins.contains("com.salesforce.oauth"));
		assertTrue("sdkinfo plugin should have been returned", sdkInfoPlugins.contains("com.salesforce.sdkinfo"));
		assertTrue("sfaccountmanager plugin should have been returned", sdkInfoPlugins.contains("com.salesforce.sfaccountmanager"));
		assertEquals("Wrong version", SalesforceSDKManager.SDK_VERSION, sdkInfo.getString("sdkVersion"));

		JSONObject sdkInfoBootConfig = sdkInfo.getJSONObject("bootConfig");
		assertEquals("Wrong bootconfig shouldAuthenticate", bootconfig.shouldAuthenticate(), sdkInfoBootConfig.getBoolean("shouldAuthenticate"));
		assertEquals("Wrong bootconfig attemptOfflineLoad", bootconfig.attemptOfflineLoad(), sdkInfoBootConfig.getBoolean("attemptOfflineLoad"));
		assertEquals("Wrong bootconfig isLocal", bootconfig.isLocal(), sdkInfoBootConfig.getBoolean("isLocal"));
		List<String> sdkInfoOAuthScopes = toList(sdkInfoBootConfig.getJSONArray("oauthScopes"));
		assertEquals("Wrong bootconfig oauthScopes", 1, sdkInfoOAuthScopes.size());
		assertTrue("Wrong bootconfig oauthScopes", sdkInfoOAuthScopes.contains("api"));
		assertEquals("Wrong bootconfig oauthRedirectURI", bootconfig.getOauthRedirectURI(), sdkInfoBootConfig.getString("oauthRedirectURI"));
		assertEquals("Wrong bootconfig remoteAccessConsumerKey", bootconfig.getRemoteAccessConsumerKey(), sdkInfoBootConfig.getString("remoteAccessConsumerKey"));
		assertEquals("Wrong bootconfig androidPushNotificationClientId", bootconfig.getPushNotificationClientId(), sdkInfoBootConfig.getString("androidPushNotificationClientId"));
		assertEquals("Wrong bootconfig startPage", "", sdkInfoBootConfig.optString("startPage")); // this is a native app
		assertEquals("Wrong bootconfig errorPage", "", sdkInfoBootConfig.optString("errorPage")); // this is a native app
	}

	/**
	 * Test for getForcePluginsFromXML
	 */
	public void testGetForcePluginsFromXML() {
		List<String> plugins = SDKInfoPlugin.getForcePluginsFromXML(getInstrumentation().getTargetContext());
		assertEquals("Wrong number of force plugins", 3, plugins.size());
		assertTrue("oauth plugin should have been returned", plugins.contains("com.salesforce.oauth"));
		assertTrue("sdkinfo plugin should have been returned", plugins.contains("com.salesforce.sdkinfo"));
		assertTrue("sfaccountmanager plugin should have been returned", plugins.contains("com.salesforce.sfaccountmanager"));
	}

	/**
	 * Helper method
	 * @param jsonArray
	 * @return
	 * @throws JSONException
	 */
	private List<String> toList(JSONArray jsonArray) throws JSONException {
		List<String> list = new ArrayList<String>(jsonArray.length());
		for (int i=0; i<jsonArray.length(); i++) {
			list.add(jsonArray.getString(i));
		}
		return list;
	}
}

The `appName` and `appVersion` validation is kinda pointless here. We can't really read this from `PackageManager`.


@@ -40,6 +40,12 @@ public abstract class MethodDescriptorView {

   public abstract List<HeaderRequestParamView> headerRequestParams();

+  @Nullable
+  public abstract String requestFormatterName();
+
+  @Nullable
+  public abstract String responseFormatterName();
+
   public boolean hasHeaderRequestParams() {
     return headerRequestParams() != null && !headerRequestParams().isEmpty();
   }

/* Copyright 2017 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.google.api.codegen.viewmodel;

import com.google.api.codegen.config.GrpcStreamingConfig.GrpcStreamingType;
import com.google.auto.value.AutoValue;
import java.util.List;
import javax.annotation.Nullable;

@AutoValue
public abstract class MethodDescriptorView {

  public abstract String requestTypeName();

  public abstract String responseTypeName();

  public abstract boolean hasResponse();

  public abstract GrpcStreamingType grpcStreamingType();

  public abstract String name();

  public abstract String protoMethodName();

  public abstract String fullServiceName();

  public abstract String transportSettingsVar();

  public abstract List<HeaderRequestParamView> headerRequestParams();

  public boolean hasHeaderRequestParams() {
    return headerRequestParams() != null && !headerRequestParams().isEmpty();
  }

  @Nullable
  public abstract HttpMethodView httpMethod();

  public static Builder newBuilder() {
    return new AutoValue_MethodDescriptorView.Builder()
        .grpcStreamingType(GrpcStreamingType.NonStreaming);
  }

  @AutoValue.Builder
  public abstract static class Builder {

    public abstract Builder requestTypeName(String name);

    public abstract Builder responseTypeName(String name);

    public abstract Builder hasResponse(boolean val);

    public abstract Builder grpcStreamingType(GrpcStreamingType val);

    public abstract Builder name(String directCallableName);

    public abstract Builder protoMethodName(String val);

    public abstract Builder fullServiceName(String val);

    public abstract Builder transportSettingsVar(String val);

    public abstract Builder headerRequestParams(List<HeaderRequestParamView> val);

    public abstract Builder httpMethod(HttpMethodView val);

    public abstract MethodDescriptorView build();
  }
}

These names aren't needed any more, right?


@@ -36,25 +36,30 @@ import org.apache.iceberg.flink.source.FlinkSource;

 /**
  * Flink Iceberg table source.
- * TODO: Implement {@link FilterableTableSource} and {@link LimitableTableSource}.
+ * TODO: Implement {@link FilterableTableSource}
  */
-public class IcebergTableSource implements StreamTableSource<RowData>, ProjectableTableSource<RowData> {
+public class IcebergTableSource
+    implements StreamTableSource<RowData>, ProjectableTableSource<RowData>, LimitableTableSource<RowData> {

   private final TableLoader loader;
   private final TableSchema schema;
   private final Map<String, String> properties;
   private final int[] projectedFields;
+  private boolean isLimitPushDown = false;
+  private long limit = -1L;

   public IcebergTableSource(TableLoader loader, TableSchema schema, Map<String, String> properties) {
-    this(loader, schema, properties, null);
+    this(loader, schema, properties, null, false, -1);
   }

   private IcebergTableSource(TableLoader loader, TableSchema schema, Map<String, String> properties,
-                             int[] projectedFields) {
+                             int[] projectedFields, boolean isLimitPushDown, long limit) {
     this.loader = loader;
     this.schema = schema;
     this.properties = properties;
     this.projectedFields = projectedFields;
+    this.isLimitPushDown = isLimitPushDown;
+    this.limit = limit;
   }

   @Override

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.apache.iceberg.flink;

import java.util.Arrays;
import java.util.Map;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.TableSchema;
import org.apache.flink.table.data.RowData;
import org.apache.flink.table.sources.FilterableTableSource;
import org.apache.flink.table.sources.LimitableTableSource;
import org.apache.flink.table.sources.ProjectableTableSource;
import org.apache.flink.table.sources.StreamTableSource;
import org.apache.flink.table.sources.TableSource;
import org.apache.flink.table.types.DataType;
import org.apache.flink.table.utils.TableConnectorUtils;
import org.apache.iceberg.flink.source.FlinkSource;

/**
 * Flink Iceberg table source.
 * TODO: Implement {@link FilterableTableSource} and {@link LimitableTableSource}.
 */
public class IcebergTableSource implements StreamTableSource<RowData>, ProjectableTableSource<RowData> {

  private final TableLoader loader;
  private final TableSchema schema;
  private final Map<String, String> properties;
  private final int[] projectedFields;

  public IcebergTableSource(TableLoader loader, TableSchema schema, Map<String, String> properties) {
    this(loader, schema, properties, null);
  }

  private IcebergTableSource(TableLoader loader, TableSchema schema, Map<String, String> properties,
                             int[] projectedFields) {
    this.loader = loader;
    this.schema = schema;
    this.properties = properties;
    this.projectedFields = projectedFields;
  }

  @Override
  public boolean isBounded() {
    return FlinkSource.isBounded(properties);
  }

  @Override
  public TableSource<RowData> projectFields(int[] fields) {
    return new IcebergTableSource(loader, schema, properties, fields);
  }

  @Override
  public DataStream<RowData> getDataStream(StreamExecutionEnvironment execEnv) {
    return FlinkSource.forRowData().env(execEnv).tableLoader(loader).project(getProjectedSchema())
        .properties(properties).build();
  }

  @Override
  public TableSchema getTableSchema() {
    return schema;
  }

  @Override
  public DataType getProducedDataType() {
    return getProjectedSchema().toRowDataType().bridgedTo(RowData.class);
  }

  private TableSchema getProjectedSchema() {
    TableSchema fullSchema = getTableSchema();
    if (projectedFields == null) {
      return fullSchema;
    } else {
      String[] fullNames = fullSchema.getFieldNames();
      DataType[] fullTypes = fullSchema.getFieldDataTypes();
      return TableSchema.builder().fields(
          Arrays.stream(projectedFields).mapToObj(i -> fullNames[i]).toArray(String[]::new),
          Arrays.stream(projectedFields).mapToObj(i -> fullTypes[i]).toArray(DataType[]::new)).build();
    }
  }

  @Override
  public String explainSource() {
    String explain = "Iceberg table: " + loader.toString();
    if (projectedFields != null) {
      explain += ", ProjectedFields: " + Arrays.toString(projectedFields);
    }
    return TableConnectorUtils.generateRuntimeName(getClass(), getTableSchema().getFieldNames()) + explain;
  }
}

Why disable the limit push down for flink table source ?


@@ -15,8 +15,10 @@

 package com.pingcap.tikv;

+import com.pingcap.tikv.key.IndexKey;
 import com.pingcap.tikv.key.Key;
 import com.pingcap.tikv.key.RowKey;
+import com.pingcap.tikv.meta.TiIndexInfo;
 import com.pingcap.tikv.meta.TiTableInfo;
 import com.pingcap.tikv.region.TiRegion;
 import java.util.ArrayList;

/*
 * Copyright 2019 PingCAP, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.pingcap.tikv;

import com.pingcap.tikv.key.Key;
import com.pingcap.tikv.key.RowKey;
import com.pingcap.tikv.meta.TiTableInfo;
import com.pingcap.tikv.region.TiRegion;
import java.util.ArrayList;
import java.util.List;

public class TiBatchWriteUtils {

  public static List<TiRegion> getRegionsByTable(TiSession session, TiTableInfo table) {
    ArrayList<TiRegion> regionList = new ArrayList<>();
    Key key = RowKey.createMin(table.getId());
    RowKey endRowKey = RowKey.createBeyondMax(table.getId());

    while (key.compareTo(endRowKey) < 0) {
      TiRegion region = session.getRegionManager().getRegionByKey(key.toByteString());
      regionList.add(region);
      key = Key.toRawKey(region.getEndKey());
    }
    return regionList;
  }
}

somewhat dangerous if region splits or changes...


@@ -29,11 +29,13 @@ import java.util.Map;
  */
 public class CommonDiscoveryProvider implements DiscoveryProvider {
   private final DiscoveryContext context;
-  private final SnippetSetRunner<Method> snippetSetRunner;
+  private final SnippetSetRunner.Generator<Method> snippetSetRunner;
   private final String snippetFileName;

   public CommonDiscoveryProvider(
-      DiscoveryContext context, SnippetSetRunner<Method> snippetSetRunner, String snippetFileName) {
+      DiscoveryContext context,
+      SnippetSetRunner.Generator<Method> snippetSetRunner,
+      String snippetFileName) {
     this.context = context;
     this.snippetSetRunner = snippetSetRunner;
     this.snippetFileName = snippetFileName;

/* Copyright 2016 Google Inc
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.google.api.codegen.discovery;

import com.google.api.codegen.DiscoveryContext;
import com.google.api.codegen.GeneratedResult;
import com.google.api.codegen.SnippetSetRunner;
import com.google.api.tools.framework.snippet.Doc;
import com.google.protobuf.Api;
import com.google.protobuf.Method;

import java.util.Collections;
import java.util.Map;

/**
 * Common DiscoveryProvider which runs code generation.
 */
public class CommonDiscoveryProvider implements DiscoveryProvider {
  private final DiscoveryContext context;
  private final SnippetSetRunner<Method> snippetSetRunner;
  private final String snippetFileName;

  public CommonDiscoveryProvider(
      DiscoveryContext context, SnippetSetRunner<Method> snippetSetRunner, String snippetFileName) {
    this.context = context;
    this.snippetSetRunner = snippetSetRunner;
    this.snippetFileName = snippetFileName;
  }

  @Override
  public Map<String, Doc> generate(Method method) {
    GeneratedResult result = snippetSetRunner.generate(method, snippetFileName, context);

    Api api = context.getApi();
    String outputRoot =
        "autogenerated/"
            + api.getName()
            + "/"
            + api.getVersion()
            + "/"
            + context.getService().getDocumentation().getOverview();
    String resultPath = outputRoot + "/" + result.getFilename();

    return Collections.singletonMap(resultPath, result.getDoc());
  }

  public static Builder newBuilder() {
    return new Builder();
  }

  public static class Builder {
    private DiscoveryContext context;
    private SnippetSetRunner<Method> snippetSetRunner;
    private String snippetFileName;

    private Builder() {}

    public Builder setContext(DiscoveryContext context) {
      this.context = context;
      return this;
    }

    public Builder setSnippetSetRunner(SnippetSetRunner<Method> snippetSetRunner) {
      this.snippetSetRunner = snippetSetRunner;
      return this;
    }

    public Builder setSnippetFileName(String snippetFileName) {
      this.snippetFileName = snippetFileName;
      return this;
    }

    public CommonDiscoveryProvider build() {
      return new CommonDiscoveryProvider(context, snippetSetRunner, snippetFileName);
    }
  }
}

Since this never changes, why not pass it into the constructor instead?


@@ -78,6 +78,10 @@ public class StringLiteralExpr extends LiteralStringValueExpr {
         return super.remove(node);
     }

+    public String asString() {
+        return value;
+    }
+
     @Override
     public StringLiteralExpr clone() {
         return (StringLiteralExpr) accept(new CloneVisitor(), null);

/*
 * Copyright (C) 2007-2010 Júlio Vilmar Gesser.
 * Copyright (C) 2011, 2013-2016 The JavaParser Team.
 *
 * This file is part of JavaParser.
 *
 * JavaParser can be used either under the terms of
 * a) the GNU Lesser General Public License as published by
 *     the Free Software Foundation, either version 3 of the License, or
 *     (at your option) any later version.
 * b) the terms of the Apache License
 *
 * You should have received a copy of both licenses in LICENCE.LGPL and
 * LICENCE.APACHE. Please refer to those files for details.
 *
 * JavaParser is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 */
package com.github.javaparser.ast.expr;

import com.github.javaparser.Range;
import com.github.javaparser.ast.AllFieldsConstructor;
import com.github.javaparser.ast.visitor.GenericVisitor;
import com.github.javaparser.ast.visitor.VoidVisitor;
import com.github.javaparser.utils.Utils;
import com.github.javaparser.ast.Node;
import com.github.javaparser.ast.visitor.CloneVisitor;
import com.github.javaparser.metamodel.StringLiteralExprMetaModel;
import com.github.javaparser.metamodel.JavaParserMetaModel;

/**
 * A literal string.
 *
"Hello World!"
 *
"\"\n"
 *
"™"
 *
"💩"
 *
 * @author Julio Vilmar Gesser
 */
public class StringLiteralExpr extends LiteralStringValueExpr {

    public StringLiteralExpr() {
        this(null, "empty");
    }

    @AllFieldsConstructor
    public StringLiteralExpr(final String value) {
        this(null, value);
    }

    /**
     * Utility method that creates a new StringLiteralExpr. Escapes EOL characters.
     */
    public static StringLiteralExpr escape(String string) {
        return new StringLiteralExpr(Utils.escapeEndOfLines(string));
    }

    public StringLiteralExpr(final Range range, final String value) {
        super(range, value);
    }

    @Override
    public <R, A> R accept(final GenericVisitor<R, A> v, final A arg) {
        return v.visit(this, arg);
    }

    @Override
    public  void accept(final VoidVisitor v, final A arg) {
        v.visit(this, arg);
    }

    @Override
    public boolean remove(Node node) {
        if (node == null)
            return false;
        return super.remove(node);
    }

    @Override
    public StringLiteralExpr clone() {
        return (StringLiteralExpr) accept(new CloneVisitor(), null);
    }

    @Override
    public StringLiteralExprMetaModel getMetaModel() {
        return JavaParserMetaModel.stringLiteralExprMetaModel;
    }
}

That's not right, is it? This should have the same (un)escaping as `asChar()`. Interesting question: so what should `StringLiteralExpr(String)` do?


@@ -191,7 +191,7 @@ final class Collections {
         if (iterable instanceof Seq) {
             return (Seq<T>) iterable;
         } else {
-            return List.ofAll(iterable);
+            return Stream.ofAll(iterable);
         }
     }
 }

/*     / \____  _    _  ____   ______  / \ ____  __    _______
 *    /  /    \/ \  / \/    \ /  /\__\/  //    \/  \  //  /\__\   JΛVΛSLΛNG
 *  _/  /  /\  \  \/  /  /\  \\__\\  \  //  /\  \ /\\/ \ /__\ \   Copyright 2014-2016 Javaslang, http://javaslang.io
 * /___/\_/  \_/\____/\_/  \_/\__\/__/\__\_/  \_//  \__/\_____/   Licensed under the Apache License, Version 2.0
 */
package javaslang.collection;

import javaslang.control.Option;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Objects;
import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.function.Supplier;

/**
 * Internal class, containing helpers.
 *
 * @author Daniel Dietrich
 * @since 2.0.0
 */
final class Collections {
    @SuppressWarnings("unchecked")
    static <C extends Traversable<T>, T> C removeAll(C collection, T element) {
        Objects.requireNonNull(element, "element is null");
        return removeAll(collection, List.of(element));
    }

    @SuppressWarnings("unchecked")
    static <C extends Traversable<T>, T> C removeAll(C collection, Iterable<? extends T> elements) {
        Objects.requireNonNull(elements, "elements is null");
        final Set<T> removed = HashSet.ofAll(elements);
        return (C) collection.filter(e -> !removed.contains(e));
    }

    @SuppressWarnings("unchecked")
    static <C extends Traversable<T>, T> C removeAll(C collection, Predicate<? super T> predicate) {
        Objects.requireNonNull(predicate, "predicate is null");
        return (C) collection.filter(predicate.negate());
    }

    @SuppressWarnings("unchecked")
    static <C extends Traversable<T>, T> C retainAll(C collection, Iterable<? extends T> elements) {
        Objects.requireNonNull(elements, "elements is null");
        final Set<T> removed = HashSet.ofAll(elements);
        return (C) collection.filter(removed::contains);
    }

    public static <T, C, R extends Iterable<T>> Map<C, R> groupBy(Traversable<T> collection, Function<? super T, ? extends C> classifier, Function<? super Iterable<T>, R> mapper) {
        Objects.requireNonNull(collection, "collection is null");
        Objects.requireNonNull(classifier, "classifier is null");
        Objects.requireNonNull(mapper, "mapper is null");

        final java.util.Map<C, Collection<T>> mutableResults = new java.util.LinkedHashMap<>();
        for (T value : collection) {
            final C key = classifier.apply(value);
            mutableResults.computeIfAbsent(key, k -> new ArrayList<>()).add(value);
        }

        Map<C, R> results = LinkedHashMap.empty();
        for (java.util.Map.Entry<C, Collection<T>> entry : mutableResults.entrySet()) {
            results = results.put(entry.getKey(), mapper.apply(entry.getValue()));
        }
        return results;
    }

    @SuppressWarnings("unchecked")
    public static <K, V, K2, U extends Map<K2, V>> U mapKeys(Map<K, V> source, U zero, Function<? super K, ? extends K2> keyMapper, BiFunction<? super V, ? super V, ? extends V> valueMerge) {
        Objects.requireNonNull(source, "source is null");
        Objects.requireNonNull(zero, "zero is null");
        Objects.requireNonNull(keyMapper, "keyMapper is null");
        Objects.requireNonNull(valueMerge, "valueMerge is null");
        return source.foldLeft(zero, (acc, entry) -> {
            final K2 k2 = keyMapper.apply(entry._1);
            final V v2 = entry._2;
            final Option<V> v1 = acc.get(k2);
            final V v = v1.isDefined() ? valueMerge.apply(v1.get(), v2) : v2;
            return (U) acc.put(k2, v);
        });
    }

    static Option<Integer> indexOption(int index) {
        return Option.when(index >= 0, index);
    }

    // checks, if the *elements* of the given iterables are equal
    static boolean areEqual(Iterable<?> iterable1, Iterable<?> iterable2) {
        final java.util.Iterator<?> iter1 = iterable1.iterator();
        final java.util.Iterator<?> iter2 = iterable2.iterator();
        while (iter1.hasNext() && iter2.hasNext()) {
            if (!Objects.equals(iter1.next(), iter2.next())) {
                return false;
            }
        }
        return iter1.hasNext() == iter2.hasNext();
    }

    // hashes the elements of an iterable
    static int hash(Iterable<?> iterable) {
        int hashCode = 1;
        for (Object o : iterable) {
            hashCode = 31 * hashCode + Objects.hashCode(o);
        }
        return hashCode;
    }

    static <T, U, C extends Iterable<U>, R extends Traversable<U>> R scanLeft(Iterable<? extends T> elements,
                                                                              U zero, BiFunction<? super U, ? super T, ? extends U> operation,
                                                                              C cumulativeResult, BiFunction<C, U, C> combiner,
                                                                              Function<C, R> finisher) {
        U acc = zero;
        cumulativeResult = combiner.apply(cumulativeResult, acc);
        for (T a : elements) {
            acc = operation.apply(acc, a);
            cumulativeResult = combiner.apply(cumulativeResult, acc);
        }
        return finisher.apply(cumulativeResult);
    }

    static <T, U, C extends Iterable<U>, R extends Traversable<U>> R scanRight(Iterable<? extends T> elements,
                                                                               U zero, BiFunction<? super T, ? super U, ? extends U> operation,
                                                                               C cumulativeResult, BiFunction<C, U, C> combiner,
                                                                               Function<C, R> finisher) {
        final Iterator<? extends T> reversedElements = seq(elements).reverseIterator(); // TODO a List will be reversed too many times this way
        return scanLeft(reversedElements, zero, (u, t) -> operation.apply(t, u), cumulativeResult, combiner, finisher);
    }

    @SuppressWarnings("unchecked")
    static <T, S extends Seq<T>> Iterator<S> crossProduct(S empty, S seq, int power) {
        if (power < 0) {
            return Iterator.empty();
        }
        return Iterator.range(0, power)
                       .foldLeft(Iterator.of(empty), (product, ignored) -> product.flatMap(el -> seq.map(t -> (S) el.append(t))));
    }

    static <C extends Traversable<T>, T> C tabulate(int n, Function<? super Integer, ? extends T> f, C empty, Function<T[], C> of) {
        Objects.requireNonNull(f, "f is null");
        Objects.requireNonNull(empty, "empty is null");
        Objects.requireNonNull(of, "of is null");
        if (n <= 0) {
            return empty;
        } else {
            @SuppressWarnings("unchecked")
            final T[] elements = (T[]) new Object[n];
            for (int i = 0; i < n; i++) {
                elements[i] = f.apply(i);
            }
            return of.apply(elements);
        }
    }

    static <C extends Traversable<T>, T> C fill(int n, Supplier<? extends T> s, C empty, Function<T[], C> of) {
        Objects.requireNonNull(s, "s is null");
        Objects.requireNonNull(empty, "empty is null");
        Objects.requireNonNull(of, "of is null");
        return tabulate(n, anything -> s.get(), empty, of);
    }

    static <T> Iterator<T> tabulate(int n, Function<? super Integer, ? extends T> f) {
        Objects.requireNonNull(f, "f is null");
        if (n <= 0) {
            return Iterator.empty();
        } else {
            return new AbstractIterator<T>() {

                int i = 0;

                @Override
                public boolean hasNext() {
                    return i < n;
                }

                @Override
                protected T getNext() {
                    return f.apply(i++);
                }
            };
        }
    }

    static <T> Iterator<T> fill(int n, Supplier<? extends T> supplier) {
        Objects.requireNonNull(supplier, "supplier is null");
        return tabulate(n, ignored -> supplier.get());
    }

    @SuppressWarnings("unchecked")
    static <T> Seq<T> seq(Iterable<? extends T> iterable) {
        if (iterable instanceof Seq) {
            return (Seq<T>) iterable;
        } else {
            return List.ofAll(iterable);
        }
    }
}

I'm not sure about this, please check the usages. It's only used currently to reverse it, maybe we should eliminate this method completely instead.


@@ -34,4 +34,6 @@ public interface DefinitionConst {
   String VERSION_RULE_LATEST = "latest";

   String VERSION_RULE_ALL = "0.0.0+";
+
+  String DEFAULT_REVISION = "0";
 }

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package io.servicecomb.serviceregistry.definition;

public interface DefinitionConst {

  String CONFIG_QUALIFIED_INSTANCE_ENVIRONMENT_KEY = "instance_description.environment";

  String CONFIG_ALLOW_CROSS_APP_KEY = "allowCrossApp";

  String DEFAULT_APPLICATION_ID = "default";

  String DEFAULT_MICROSERVICE_VERSION = "1.0.0";

  String DEFAULT_STAGE = "prod";

  String DEFAULT_INSTANCE_ENVIRONMENT = "production";

  String VERSION_RULE_LATEST = "latest";

  String VERSION_RULE_ALL = "0.0.0+";
}

there is no "DEFAULT_REVISION" logic, no need to define this.


@@ -38,25 +38,21 @@ public class PrivateTransactionValidator {
       final PrivateTransaction transaction,
       final Long accountNonce,
       final boolean allowFutureNonces) {
-    LOG.debug("Validating private transaction fields of {}", transaction.getHash());
+    LOG.debug("Validating private transaction {}", transaction);
     final ValidationResult<TransactionInvalidReason> privateFieldsValidationResult =
         validatePrivateTransactionFields(transaction);
     if (!privateFieldsValidationResult.isValid()) {
       LOG.debug(
-          "Private Transaction fields are invalid {}, {}",
-          transaction.getHash(),
+          "Private Transaction fields are invalid {}",
           privateFieldsValidationResult.getErrorMessage());
       return privateFieldsValidationResult;
     }

-    LOG.debug("Validating the signature of Private Transaction {} ", transaction.getHash());
-
     final ValidationResult<TransactionValidator.TransactionInvalidReason>
         signatureValidationResult = validateTransactionSignature(transaction);
     if (!signatureValidationResult.isValid()) {
       LOG.debug(
-          "Private Transaction {}, failed validation {}, {}",
-          transaction.getHash(),
+          "Private Transaction failed signature validation {}, {}",
           signatureValidationResult.getInvalidReason(),
           signatureValidationResult.getErrorMessage());
       return signatureValidationResult;

/*
 * Copyright ConsenSys AG.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
 * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations under the License.
 *
 * SPDX-License-Identifier: Apache-2.0
 */
package org.hyperledger.besu.ethereum.privacy;

import org.hyperledger.besu.ethereum.mainnet.TransactionValidator;
import org.hyperledger.besu.ethereum.mainnet.TransactionValidator.TransactionInvalidReason;
import org.hyperledger.besu.ethereum.mainnet.ValidationResult;

import java.math.BigInteger;
import java.util.Optional;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

public class PrivateTransactionValidator {

  private static final Logger LOG = LogManager.getLogger();

  private final Optional<BigInteger> chainId;

  public PrivateTransactionValidator(final Optional<BigInteger> chainId) {
    this.chainId = chainId;
  }

  public ValidationResult<TransactionValidator.TransactionInvalidReason> validate(
      final PrivateTransaction transaction,
      final Long accountNonce,
      final boolean allowFutureNonces) {
    LOG.debug("Validating private transaction fields of {}", transaction.getHash());
    final ValidationResult<TransactionInvalidReason> privateFieldsValidationResult =
        validatePrivateTransactionFields(transaction);
    if (!privateFieldsValidationResult.isValid()) {
      LOG.debug(
          "Private Transaction fields are invalid {}, {}",
          transaction.getHash(),
          privateFieldsValidationResult.getErrorMessage());
      return privateFieldsValidationResult;
    }

    LOG.debug("Validating the signature of Private Transaction {} ", transaction.getHash());

    final ValidationResult<TransactionValidator.TransactionInvalidReason>
        signatureValidationResult = validateTransactionSignature(transaction);
    if (!signatureValidationResult.isValid()) {
      LOG.debug(
          "Private Transaction {}, failed validation {}, {}",
          transaction.getHash(),
          signatureValidationResult.getInvalidReason(),
          signatureValidationResult.getErrorMessage());
      return signatureValidationResult;
    }

    final long transactionNonce = transaction.getNonce();

    LOG.debug("Validating actual nonce {}, with expected nonce {}", transactionNonce, accountNonce);

    if (accountNonce > transactionNonce) {
      final String errorMessage =
          String.format(
              "Private Transaction nonce %s, is lower than sender account nonce %s.",
              transactionNonce, accountNonce);
      LOG.debug(errorMessage);
      return ValidationResult.invalid(
          TransactionValidator.TransactionInvalidReason.PRIVATE_NONCE_TOO_LOW, errorMessage);
    }

    if (!allowFutureNonces && accountNonce != transactionNonce) {
      final String errorMessage =
          String.format(
              "Private Transaction nonce %s, does not match sender account nonce %s.",
              transactionNonce, accountNonce);
      LOG.debug(errorMessage);
      return ValidationResult.invalid(
          TransactionValidator.TransactionInvalidReason.INCORRECT_PRIVATE_NONCE, errorMessage);
    }

    return ValidationResult.valid();
  }

  private ValidationResult<TransactionValidator.TransactionInvalidReason>
      validatePrivateTransactionFields(final PrivateTransaction privateTransaction) {
    if (!privateTransaction.getValue().isZero()) {
      return ValidationResult.invalid(
          TransactionValidator.TransactionInvalidReason.PRIVATE_VALUE_NOT_ZERO);
    }
    if (!privateTransaction.getRestriction().equals(Restriction.RESTRICTED)) {
      return ValidationResult.invalid(
          TransactionValidator.TransactionInvalidReason.PRIVATE_UNIMPLEMENTED_TRANSACTION_TYPE);
    }

    return ValidationResult.valid();
  }

  private ValidationResult<TransactionValidator.TransactionInvalidReason>
      validateTransactionSignature(final PrivateTransaction transaction) {
    if (chainId.isPresent()
        && (transaction.getChainId().isPresent() && !transaction.getChainId().equals(chainId))) {
      return ValidationResult.invalid(
          TransactionValidator.TransactionInvalidReason.WRONG_CHAIN_ID,
          String.format(
              "Transaction was meant for chain id %s, not this chain id %s",
              transaction.getChainId().get(), chainId.get()));
    }

    if (chainId.isEmpty() && transaction.getChainId().isPresent()) {
      return ValidationResult.invalid(
          TransactionValidator.TransactionInvalidReason.REPLAY_PROTECTED_SIGNATURES_NOT_SUPPORTED,
          "Replay protection (chainId) is not supported");
    }

    // org.bouncycastle.math.ec.ECCurve.AbstractFp.decompressPoint throws an
    // IllegalArgumentException for "Invalid point compression" for bad signatures.
    try {
      transaction.getSender();
    } catch (final IllegalArgumentException e) {
      return ValidationResult.invalid(
          TransactionValidator.TransactionInvalidReason.INVALID_SIGNATURE,
          "Sender could not be extracted from transaction signature");
    }

    return ValidationResult.valid();
  }
}

"Validating private transaction corresponding to PMT {}" ?


@@ -17,20 +17,17 @@
 package io.servicecomb.transport.rest.servlet;

 import io.servicecomb.common.rest.RestProducerInvocation;
-import io.servicecomb.common.rest.definition.RestOperationMeta;
 import io.servicecomb.common.rest.filter.HttpServerFilter;
 import io.servicecomb.core.definition.OperationMeta;
 import io.servicecomb.foundation.vertx.http.StandardHttpServletRequestEx;

 public class RestServletProducerInvocation extends RestProducerInvocation {
   @Override
-  protected RestOperationMeta findRestOperation() {
-    RestOperationMeta restOperationMeta = super.findRestOperation();
+  protected void findRestOperation() {
+    super.findRestOperation();

     boolean cacheRequest = collectCacheRequest(restOperationMeta.getOperationMeta());
     ((StandardHttpServletRequestEx) requestEx).setCacheRequest(cacheRequest);
-
-    return restOperationMeta;
   }

   protected boolean collectCacheRequest(OperationMeta operationMeta) {

/*
 * Copyright 2017 Huawei Technologies Co., Ltd
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package io.servicecomb.transport.rest.servlet;

import io.servicecomb.common.rest.RestProducerInvocation;
import io.servicecomb.common.rest.definition.RestOperationMeta;
import io.servicecomb.common.rest.filter.HttpServerFilter;
import io.servicecomb.core.definition.OperationMeta;
import io.servicecomb.foundation.vertx.http.StandardHttpServletRequestEx;

public class RestServletProducerInvocation extends RestProducerInvocation {
  @Override
  protected RestOperationMeta findRestOperation() {
    RestOperationMeta restOperationMeta = super.findRestOperation();

    boolean cacheRequest = collectCacheRequest(restOperationMeta.getOperationMeta());
    ((StandardHttpServletRequestEx) requestEx).setCacheRequest(cacheRequest);

    return restOperationMeta;
  }

  protected boolean collectCacheRequest(OperationMeta operationMeta) {
    for (HttpServerFilter filter : httpServerFilters) {
      if (filter.needCacheRequest(operationMeta)) {
        return true;
      }
    }
    return false;
  }
}


@@ -58,14 +58,16 @@ public final class BaselineReleaseCompatibility extends AbstractBaselinePlugin {

         @Override
         public Iterable<String> asArguments() {
-            JavaVersion jdkVersion = getJdkVersion(javaCompile);
-
-            if (!supportsReleaseFlag(jdkVersion)) {
+            if (javaCompile.getProject().getPlugins().hasPlugin(BaselineJavaVersion.class)) {
+                return Collections.emptyList();
+            }
+            JavaVersion compilerVersion = JavaVersion.current();
+            if (!compilerVersion.isJava9Compatible()) {
                 log.debug(
                         "BaselineReleaseCompatibility is a no-op for {} in {} as {} doesn't support --release",
                         javaCompile.getName(),
                         javaCompile.getProject(),
-                        jdkVersion);
+                        compilerVersion);
                 return Collections.emptyList();
             }

/*
 * (c) Copyright 2019 Palantir Technologies Inc. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.palantir.baseline.plugins;

import com.google.common.collect.ImmutableList;
import java.util.Collections;
import java.util.Optional;
import org.gradle.api.JavaVersion;
import org.gradle.api.Project;
import org.gradle.api.tasks.compile.JavaCompile;
import org.gradle.jvm.toolchain.JavaCompiler;
import org.gradle.jvm.toolchain.JavaInstallationMetadata;
import org.gradle.process.CommandLineArgumentProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * When using JDK 9+ to compile with a targetCompatibility less than JDK 9, this plugin adds compiler arguments per JEP 247 to explicitly set the target JDK platform API to maintain binary
 * compatibility.
 *
 *
See also Gradle JDK release issue.
 */
public final class BaselineReleaseCompatibility extends AbstractBaselinePlugin {
    private static final Logger log = LoggerFactory.getLogger(BaselineReleaseCompatibility.class);

    @Override
    public void apply(Project project) {
        this.project = project;

        project.getTasks().withType(JavaCompile.class).configureEach(javaCompile -> {
            javaCompile.getOptions().getCompilerArgumentProviders().add(new ReleaseFlagProvider(javaCompile));
        });
    }

    // using a lazy argument provider is crucial because otherwise we'd try to read sourceCompat / targetCompat
    // before the user has even set it in their build.gradle!
    private static final class ReleaseFlagProvider implements CommandLineArgumentProvider {
        private final JavaCompile javaCompile;

        private ReleaseFlagProvider(JavaCompile javaCompile) {
            this.javaCompile = javaCompile;
        }

        @Override
        public Iterable<String> asArguments() {
            JavaVersion jdkVersion = getJdkVersion(javaCompile);

            if (!supportsReleaseFlag(jdkVersion)) {
                log.debug(
                        "BaselineReleaseCompatibility is a no-op for {} in {} as {} doesn't support --release",
                        javaCompile.getName(),
                        javaCompile.getProject(),
                        jdkVersion);
                return Collections.emptyList();
            }

            // The java compiler does not allow using --add-exports in combination with --release
            if (javaCompile.getOptions().getCompilerArgs().stream().anyMatch(arg -> arg.startsWith("--add-exports"))) {
                log.debug(
                        "BaselineReleaseCompatibility is a no-op for {} in {} as --add-exports flag is also used",
                        javaCompile.getName(),
                        javaCompile.getProject());
                return Collections.emptyList();
            }

            Optional<JavaVersion> taskTarget =
                    Optional.ofNullable(javaCompile.getTargetCompatibility()).map(JavaVersion::toVersion);

            if (!taskTarget.isPresent()) {
                log.debug(
                        "BaselineReleaseCompatibility is a no-op for {} in {} as no targetCompatibility is set",
                        javaCompile.getName(),
                        javaCompile.getProject());
                return Collections.emptyList();
            }
            JavaVersion target = taskTarget.get();

            if (jdkVersion.compareTo(target) <= 0) {
                log.debug(
                        "BaselineReleaseCompatibility is a no-op for {} in {} as targetCompatibility is higher",
                        javaCompile.getName(),
                        javaCompile.getProject());
                return Collections.emptyList();
            }

            return ImmutableList.of("--release", target.getMajorVersion());
        }

        // The --release flag was added in Java 9: https://openjdk.java.net/jeps/247
        private static boolean supportsReleaseFlag(JavaVersion jdkVersion) {
            return jdkVersion.isJava9Compatible();
        }
    }

    private static JavaVersion getJdkVersion(JavaCompile javaCompile) {
        return javaCompile
                .getJavaCompiler()
                .map(JavaCompiler::getMetadata)
                .map(JavaInstallationMetadata::getLanguageVersion)
                .map(version -> JavaVersion.toVersion(version.asInt()))
                // Fallback to current java version if toolchain is not configured
                .getOrElse(JavaVersion.current());
    }
}

nit: Can we add a log here that this is also a no-op because BaselineJavaVersion is used?


@@ -13,14 +13,14 @@
  * License for the specific language governing permissions and limitations under
  * the License.
  */
-
 package azkaban.execapp.event;

 import azkaban.executor.Status;

+
 public class BlockingStatus {

-  private static final long WAIT_TIME = 5 * 60 * 1000;
+  private static final long WAIT_TIME = 300000;  // 5 * 60 * 1000
   private final int execId;
   private final String jobId;
   private Status status;

/*
 * Copyright 2012 LinkedIn Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

package azkaban.execapp.event;

import azkaban.executor.Status;

public class BlockingStatus {

  private static final long WAIT_TIME = 5 * 60 * 1000;
  private final int execId;
  private final String jobId;
  private Status status;

  public BlockingStatus(final int execId, final String jobId, final Status initialStatus) {
    this.execId = execId;
    this.jobId = jobId;
    this.status = initialStatus;
  }

  public Status blockOnFinishedStatus() {
    if (this.status == null) {
      return null;
    }

    while (!Status.isStatusFinished(this.status)) {
      synchronized (this) {
        try {
          this.wait(WAIT_TIME);
        } catch (final InterruptedException e) {
        }
      }
    }

    return this.status;
  }

  public Status viewStatus() {
    return this.status;
  }

  public void unblock() {
    synchronized (this) {
      this.notifyAll();
    }
  }

  public void changeStatus(final Status status) {
    synchronized (this) {
      this.status = status;
      if (Status.isStatusFinished(status)) {
        unblock();
      }
    }
  }

  public int getExecId() {
    return this.execId;
  }

  public String getJobId() {
    return this.jobId;
  }
}


@@ -45,7 +45,7 @@ import org.slf4j.LoggerFactory;
 public class QuartzScheduler {

   //Unless specified, all Quartz jobs's identities comes with the default job name.
-  private static final String DEFAULT_JOB_NAME = "job1";
+  public static final String DEFAULT_JOB_NAME = "job1";
   private static final Logger logger = LoggerFactory.getLogger(QuartzScheduler.class);
   private Scheduler scheduler = null;

/*
 * Copyright 2017 LinkedIn Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

package azkaban.scheduler;

import static azkaban.ServiceProvider.SERVICE_PROVIDER;
import static java.util.Objects.requireNonNull;

import azkaban.Constants.ConfigurationKeys;
import azkaban.utils.Props;
import java.util.Set;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.quartz.CronExpression;
import org.quartz.CronScheduleBuilder;
import org.quartz.JobBuilder;
import org.quartz.JobDetail;
import org.quartz.JobKey;
import org.quartz.Scheduler;
import org.quartz.SchedulerException;
import org.quartz.Trigger;
import org.quartz.TriggerBuilder;
import org.quartz.impl.StdSchedulerFactory;
import org.quartz.impl.matchers.GroupMatcher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Manages Quartz schedules. Azkaban regards QuartzJob and QuartzTrigger as an one-to-one mapping.
 */
@Singleton
public class QuartzScheduler {

  //Unless specified, all Quartz jobs's identities comes with the default job name.
  private static final String DEFAULT_JOB_NAME = "job1";
  private static final Logger logger = LoggerFactory.getLogger(QuartzScheduler.class);
  private Scheduler scheduler = null;

  @Inject
  public QuartzScheduler(final Props azProps) throws SchedulerException {
    if (!azProps.getBoolean(ConfigurationKeys.ENABLE_QUARTZ, false)) {
      return;
    }
    final StdSchedulerFactory schedulerFactory =
        new StdSchedulerFactory(azProps.toProperties());
    this.scheduler = schedulerFactory.getScheduler();

    // Currently Quartz only support internal job schedules. When we migrate to User Production
    // flows, we need to construct a Guice-Free JobFactory for use.
    this.scheduler.setJobFactory(SERVICE_PROVIDER.getInstance(SchedulerJobFactory.class));
  }

  public void start() {
    try {
      this.scheduler.start();
    } catch (final SchedulerException e) {
      logger.error("Error starting Quartz scheduler: ", e);
    }
    logger.info("Quartz Scheduler started.");
  }

  public void cleanup() {
    logger.info("Cleaning up schedules in scheduler");
    try {
      this.scheduler.clear();
    } catch (final SchedulerException e) {
      logger.error("Exception clearing scheduler: ", e);
    }
  }

  public void pause() {
    logger.info("pausing all schedules in Quartz");
    try {
      this.scheduler.pauseAll();
    } catch (final SchedulerException e) {
      logger.error("Exception pausing scheduler: ", e);
    }
  }

  public void resume() {
    logger.info("resuming all schedules in Quartz");
    try {
      this.scheduler.resumeAll();
    } catch (final SchedulerException e) {
      logger.error("Exception resuming scheduler: ", e);
    }
  }

  public void shutdown() {
    logger.info("Shutting down scheduler");
    try {
      this.scheduler.shutdown();
    } catch (final SchedulerException e) {
      logger.error("Exception shutting down scheduler: ", e);
    }
  }

  public void unregisterJob(final String groupName) throws SchedulerException {
    if (!ifJobExist(groupName)) {
      logger.warn("can not find job with " + groupName + " in quartz.");
    } else {
      this.scheduler.deleteJob(new JobKey(DEFAULT_JOB_NAME, groupName));
    }
  }

  /**
   * Only cron schedule register is supported.
   *
   * @param cronExpression the cron schedule for this job
   * @param jobDescription Regarding QuartzJobDescription#groupName, in order to guarantee no
   * duplicate quartz schedules, we design the naming convention depending on use cases:

   *
User flow schedule: we use {@link org.quartz.JobKey#JobKey} to represent the identity of a
   * flow's schedule. The format follows "$projectID_$flowName" to guarantee no duplicates.
   *
Quartz schedule for AZ internal use: the groupName should start with letters,
   * rather than
   * number, which is the first case.

   */
  public void registerJob(final String cronExpression, final QuartzJobDescription jobDescription)
      throws SchedulerException {

    requireNonNull(jobDescription, "jobDescription is null");

    // Not allowed to register duplicate job name.
    if (ifJobExist(jobDescription.getGroupName())) {
      throw new SchedulerException(
          "can not register existing job " + jobDescription.getGroupName());
    }

    if (!CronExpression.isValidExpression(cronExpression)) {
      throw new SchedulerException(
          "The cron expression string <" + cronExpression + "> is not valid.");
    }

    // TODO kunkun-tang: we will modify this when we start supporting multi schedules per flow.
    final JobDetail job = JobBuilder.newJob(jobDescription.getJobClass())
        .withIdentity(DEFAULT_JOB_NAME, jobDescription.getGroupName()).build();

    // Add external dependencies to Job Data Map.
    job.getJobDataMap().putAll(jobDescription.getContextMap());

    // TODO kunkun-tang: Need management code to deal with different misfire policy
    final Trigger trigger = TriggerBuilder
        .newTrigger()
        .withSchedule(
            CronScheduleBuilder.cronSchedule(cronExpression)
                .withMisfireHandlingInstructionFireAndProceed()
//            .withMisfireHandlingInstructionDoNothing()
//            .withMisfireHandlingInstructionIgnoreMisfires()
        )
        .build();

    this.scheduler.scheduleJob(job, trigger);
    logger.info("Quartz Schedule with jobDetail " + job.getDescription() + " is registered.");
  }

  public boolean ifJobExist(final String groupName) throws SchedulerException {
    final Set<JobKey> jobKeySet = this.scheduler.getJobKeys(GroupMatcher.jobGroupEquals(groupName));
    return jobKeySet != null && jobKeySet.size() > 0;
  }

  public Scheduler getScheduler() {
    return this.scheduler;
  }
}

This should not be called Flow Trigger. FlowTrigger should has its own flowTrigger job name, for instance, "flowtrigger"


@@ -118,6 +118,26 @@ public class TestSnapshotTableProcedure extends SparkExtensionsTestBase {
         sql("SELECT * FROM %s ORDER BY id", tableName));
   }

+  @Test
+  public void testDropTable() throws IOException {
+    String location = temp.newFolder().toString();
+    sql("CREATE TABLE %s (id bigint NOT NULL, data string) USING parquet LOCATION '%s'", sourceName, location);
+    sql("INSERT INTO TABLE %s VALUES (1, 'a')", sourceName);
+
+    Object result = scalarSql("CALL %s.system.snapshot('%s', '%s')", catalogName, sourceName, tableName);
+    Assert.assertEquals("Should have added one file", 1L, result);
+
+    assertEquals("Should have expected rows",
+        ImmutableList.of(row(1L, "a")),
+        sql("SELECT * FROM %s", tableName));
+
+    sql("DROP TABLE %s", tableName);
+
+    assertEquals("Source table should be intact",
+        ImmutableList.of(row(1L, "a")),
+        sql("SELECT * FROM %s", sourceName));
+  }
+
   @Test
   public void testInvalidSnapshotsCases() throws IOException {
     String location = temp.newFolder().toString();

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.apache.iceberg.spark.extensions;

import java.io.IOException;
import java.util.Map;
import org.apache.iceberg.AssertHelpers;
import org.apache.iceberg.Table;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
import org.apache.spark.sql.AnalysisException;
import org.junit.After;
import org.junit.Assert;
import org.junit.Assume;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;

public class TestSnapshotTableProcedure extends SparkExtensionsTestBase {
  private static final String sourceName = "spark_catalog.default.source";
  // Currently we can only Snapshot only out of the Spark Session Catalog

  public TestSnapshotTableProcedure(String catalogName, String implementation, Map<String, String> config) {
    super(catalogName, implementation, config);
  }

  @Rule
  public TemporaryFolder temp = new TemporaryFolder();

  @After
  public void removeTables() {
    sql("DROP TABLE IF EXISTS %s", tableName);
    sql("DROP TABLE IF EXISTS %S", sourceName);
  }

  @Test
  public void testSnapshot() throws IOException {
    String location = temp.newFolder().toString();
    sql("CREATE TABLE %s (id bigint NOT NULL, data string) USING parquet LOCATION '%s'", sourceName, location);
    sql("INSERT INTO TABLE %s VALUES (1, 'a')", sourceName);
    Object result = scalarSql("CALL %s.system.snapshot('%s', '%s')", catalogName, sourceName, tableName);

    Assert.assertEquals("Should have added one file", 1L, result);

    Table createdTable = validationCatalog.loadTable(tableIdent);
    String tableLocation = createdTable.location();
    Assert.assertNotEquals("Table should not have the original location", location, tableLocation);

    sql("INSERT INTO TABLE %s VALUES (1, 'a')", tableName);

    assertEquals("Should have expected rows",
        ImmutableList.of(row(1L, "a"), row(1L, "a")),
        sql("SELECT * FROM %s ORDER BY id", tableName));
  }

  @Test
  public void testSnapshotWithProperties() throws IOException {
    String location = temp.newFolder().toString();
    sql("CREATE TABLE %s (id bigint NOT NULL, data string) USING parquet LOCATION '%s'", sourceName, location);
    sql("INSERT INTO TABLE %s VALUES (1, 'a')", sourceName);
    Object result = scalarSql(
        "CALL %s.system.snapshot(source_table => '%s', table => '%s', properties => map('foo','bar'))",
        catalogName, sourceName, tableName);

    Assert.assertEquals("Should have added one file", 1L, result);

    Table createdTable = validationCatalog.loadTable(tableIdent);

    String tableLocation = createdTable.location();
    Assert.assertNotEquals("Table should not have the original location", location, tableLocation);

    Map<String, String> props = createdTable.properties();
    Assert.assertEquals("Should have extra property set", "bar", props.get("foo"));

    sql("INSERT INTO TABLE %s VALUES (1, 'a')", tableName);

    assertEquals("Should have expected rows",
        ImmutableList.of(row(1L, "a"), row(1L, "a")),
        sql("SELECT * FROM %s ORDER BY id", tableName));
  }

  @Test
  public void testSnapshotWithAlternateLocation() throws IOException {
    Assume.assumeTrue("No Snapshoting with Alternate locations with Hadoop Catalogs", !catalogName.contains("hadoop"));
    String location = temp.newFolder().toString();
    String snapshotLocation = temp.newFolder().toString();
    sql("CREATE TABLE %s (id bigint NOT NULL, data string) USING parquet LOCATION '%s'", sourceName, location);
    sql("INSERT INTO TABLE %s VALUES (1, 'a')", sourceName);
    Object[] result = sql(
        "CALL %s.system.snapshot(source_table => '%s', table => '%s', location => '%s')",
        catalogName, sourceName, tableName, snapshotLocation).get(0);

    Assert.assertEquals("Should have added one file", 1L, result[0]);

    String storageLocation = validationCatalog.loadTable(tableIdent).location();
    Assert.assertEquals("Snapshot should be made at specified location", snapshotLocation, storageLocation);

    sql("INSERT INTO TABLE %s VALUES (1, 'a')", tableName);

    assertEquals("Should have expected rows",
        ImmutableList.of(row(1L, "a"), row(1L, "a")),
        sql("SELECT * FROM %s ORDER BY id", tableName));
  }

  @Test
  public void testInvalidSnapshotsCases() throws IOException {
    String location = temp.newFolder().toString();
    sql("CREATE TABLE %s (id bigint NOT NULL, data string) USING parquet LOCATION '%s'", sourceName, location);

    AssertHelpers.assertThrows("Should reject calls without all required args",
        AnalysisException.class, "Missing required parameters",
        () -> sql("CALL %s.system.snapshot('foo')", catalogName));

    AssertHelpers.assertThrows("Should reject calls with invalid arg types",
        AnalysisException.class, "Wrong arg type",
        () -> sql("CALL %s.system.snapshot('n', 't', map('foo', 'bar'))", catalogName));

    AssertHelpers.assertThrows("Should reject calls with invalid map args",
        AnalysisException.class, "cannot resolve 'map",
        () -> sql("CALL %s.system.snapshot('%s', 'fable', 'loc', map(2, 1, 1))", catalogName, sourceName));

    AssertHelpers.assertThrows("Should reject calls with empty table identifier",
        IllegalArgumentException.class, "Cannot handle an empty identifier",
        () -> sql("CALL %s.system.snapshot('', 'dest')", catalogName));

    AssertHelpers.assertThrows("Should reject calls with empty table identifier",
        IllegalArgumentException.class, "Cannot handle an empty identifier",
        () -> sql("CALL %s.system.snapshot('src', '')", catalogName));
  }
}

Do we want to assert that the metadata files for the snapshot table are removed?


@@ -39,11 +39,8 @@ public class ViewsConfigurationLibrary {

     @Bean
     public ViewLibrary viewLibrary() {
-        return new ViewLibrary(
-                applicationName,
-                themeConfiguration.themeManager(),
-                phoenicisGlobalConfiguration.objectMapper()
-        );
+        return new ViewLibrary(applicationName, themeConfiguration.themeManager(),
+                phoenicisGlobalConfiguration.objectMapper());
     }

     @Bean

/*
 * Copyright (C) 2015-2017 PÂRIS Quentin
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */

package org.phoenicis.javafx.views.mainwindow.library;

import org.phoenicis.configuration.PhoenicisGlobalConfiguration;
import org.phoenicis.javafx.views.common.ThemeConfiguration;
import org.phoenicis.javafx.views.mainwindow.console.ConsoleTabFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

@Configuration
public class ViewsConfigurationLibrary {
    @Value("${application.name}")
    private String applicationName;

    @Autowired
    private ThemeConfiguration themeConfiguration;

    @Autowired
    private PhoenicisGlobalConfiguration phoenicisGlobalConfiguration;

    @Bean
    public ViewLibrary viewLibrary() {
        return new ViewLibrary(
                applicationName,
                themeConfiguration.themeManager(),
                phoenicisGlobalConfiguration.objectMapper()
        );
    }

    @Bean
    public ConsoleTabFactory consoleTabFactory() {
        return new ConsoleTabFactory();
    }
}

It would be great if the code formatter kept the other way of presenting beans parameters. It is a lot more readable


@@ -32,6 +32,7 @@ import org.apache.tuweni.bytes.Bytes32;

 public class KeyPairUtil {
   private static final Logger LOG = LogManager.getLogger();
+  private static final EllipticCurveSignature ELLIPTIC_CURVE_SIGNATURE = EllipticCurveSignatureFactory.getInstance();

   public static String loadResourceFile(final String resourcePath) {
     try {

/*
 * Copyright ConsenSys AG.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
 * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations under the License.
 *
 * SPDX-License-Identifier: Apache-2.0
 */
package org.hyperledger.besu.crypto;

import static java.nio.file.StandardCopyOption.ATOMIC_MOVE;
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;

import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;

import com.google.common.io.Resources;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.tuweni.bytes.Bytes32;

public class KeyPairUtil {
  private static final Logger LOG = LogManager.getLogger();

  public static String loadResourceFile(final String resourcePath) {
    try {
      URL path = KeyPairUtil.class.getClassLoader().getResource(resourcePath);
      return Resources.toString(path, StandardCharsets.UTF_8).trim();
    } catch (Exception e) {
      throw new RuntimeException("Unable to load resource: " + resourcePath, e);
    }
  }

  public static SECP256K1.KeyPair loadKeyPairFromResource(final String resourcePath) {
    final SECP256K1.KeyPair keyPair;
    String keyData = loadResourceFile(resourcePath);
    if (keyData == null || keyData.isEmpty()) {
      throw new IllegalArgumentException("Unable to load resource: " + resourcePath);
    }
    SECP256K1.PrivateKey privateKey = SECP256K1.PrivateKey.create(Bytes32.fromHexString((keyData)));
    keyPair = SECP256K1.KeyPair.create(privateKey);

    LOG.info("Loaded keyPair {} from {}", keyPair.getPublicKey().toString(), resourcePath);
    return keyPair;
  }

  public static SECP256K1.KeyPair loadKeyPair(final File keyFile) {

    final SECP256K1.KeyPair key;
    if (keyFile.exists()) {

      key = load(keyFile);
      LOG.info(
          "Loaded public key {} from {}", key.getPublicKey().toString(), keyFile.getAbsolutePath());
    } else {
      key = SECP256K1.KeyPair.generate();
      try {
        storeKeyPair(key, keyFile);
      } catch (IOException e) {
        throw new IllegalArgumentException("Cannot store generated private key.");
      }
      LOG.info(
          "Generated new public key {} and stored it to {}",
          key.getPublicKey().toString(),
          keyFile.getAbsolutePath());
    }
    return key;
  }

  public static SECP256K1.KeyPair loadKeyPair(final Path homeDirectory) {
    return loadKeyPair(getDefaultKeyFile(homeDirectory));
  }

  public static File getDefaultKeyFile(final Path homeDirectory) {
    return homeDirectory.resolve("key").toFile();
  }

  public static SECP256K1.KeyPair load(final File file) {
    return SECP256K1.KeyPair.create(loadPrivateKey(file));
  }

  static SECP256K1.PrivateKey loadPrivateKey(final File file) {
    try {
      final List<String> info = Files.readAllLines(file.toPath());
      if (info.size() != 1) {
        throw new IllegalArgumentException("Supplied file does not contain valid keyPair pair.");
      }
      return SECP256K1.PrivateKey.create(Bytes32.fromHexString((info.get(0))));
    } catch (IOException ex) {
      throw new IllegalArgumentException("Supplied file does not contain valid keyPair pair.");
    }
  }

  static void storeKeyPair(final SECP256K1.KeyPair keyKair, final File file) throws IOException {
    final File privateKeyDir = file.getParentFile();
    privateKeyDir.mkdirs();
    final Path tempPath = Files.createTempFile(privateKeyDir.toPath(), ".tmp", "");
    Files.write(
        tempPath,
        keyKair.getPrivateKey().getEncodedBytes().toString().getBytes(StandardCharsets.UTF_8));
    Files.move(tempPath, file.toPath(), REPLACE_EXISTING, ATOMIC_MOVE);
  }
}

Fixing this as a static final field makes runtime or startup determination of the curve problematic. The config would be locked in at _class loading_ time. We should use the method on the factory instead. Locking the factory is one thing, locking what the factory produces is something entirely different. If we had a lazy initialized value that would also allow for later than classloading configuration. `com.google.common.base.Suppliers#memoize` is a quick path to victory here.


@@ -70,6 +70,17 @@ public interface WorldUpdater extends MutableWorldView {
     return account == null ? createAccount(address) : account;
   }

+  /**
+   * Retrieves the provided account for a sender of a transaction if it exists, or creates it if it
+   * doesn't.
+   *
+   * @param address the address of the account.
+   * @return the account {@code address}, or {@code null} if the account does not exist.
+   */
+  default EvmAccount getOrCreateSenderAccount(final Address address) {
+    return getOrCreate(address);
+  }
+
   /**
    * Retrieves the provided account, returning a modifiable object (whose updates are accumulated by
    * this updater).

/*
 * Copyright ConsenSys AG.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
 * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations under the License.
 *
 * SPDX-License-Identifier: Apache-2.0
 */
package org.hyperledger.besu.ethereum.core;

import java.util.Collection;
import java.util.Optional;

/**
 * An object that buffers updates made over a particular {@link WorldView}.
 *
 *
All changes made to this object, being it account creation/deletion or account modifications
 * through {@link MutableAccount}, are immediately reflected on this object (so for instance,
 * deleting an account and trying to get it afterwards will return {@code null}) but do not impact
 * whichever {@link WorldView} this is an updater for until the {@link #commit} method is called.
 */
public interface WorldUpdater extends MutableWorldView {

  /**
   * Creates a new account, or reset it (that is, act as if it was deleted and created anew) if it
   * already exists.
   *
   *
After this call, the account will exists and will have the provided nonce and balance. His
   * code and storage will be empty.
   *
   * @param address the address of the account to create (or reset).
   * @param nonce the nonce for created/reset account.
   * @param balance the balance for created/reset account.
   * @return the account {@code address}, which will have nonce {@code nonce}, balance {@code
   *     balance} and empty code and storage.
   */
  EvmAccount createAccount(Address address, long nonce, Wei balance);

  /**
   * Creates a new account, or reset it (that is, act as if it was deleted and created anew) if it
   * already exists.
   *
   *
This call is equivalent to {@link #createAccount(Address, long, Wei)} but defaults both the
   * nonce and balance to zero.
   *
   * @param address the address of the account to create (or reset).
   * @return the account {@code address}, which will have 0 for the nonce and balance and empty code
   *     and storage.
   */
  default EvmAccount createAccount(final Address address) {
    return createAccount(address, Account.DEFAULT_NONCE, Account.DEFAULT_BALANCE);
  }

  /**
   * Retrieves the provided account if it exists, or create it if it doesn't.
   *
   * @param address the address of the account.
   * @return the account {@code address}. If that account exists, it is returned as if by {@link
   *     #getAccount(Address)}, otherwise, it is created and returned as if by {@link
   *     #createAccount(Address)} (and thus all his fields will be zero/empty).
   */
  default EvmAccount getOrCreate(final Address address) {
    final EvmAccount account = getAccount(address);
    return account == null ? createAccount(address) : account;
  }

  /**
   * Retrieves the provided account, returning a modifiable object (whose updates are accumulated by
   * this updater).
   *
   * @param address the address of the account.
   * @return the account {@code address}, or {@code null} if the account does not exist.
   */
  EvmAccount getAccount(Address address);

  /**
   * Deletes the provided account.
   *
   * @param address the address of the account to delete. If that account doesn't exists prior to
   *     this call, this is a no-op.
   */
  void deleteAccount(Address address);

  /**
   * Returns the accounts that have been touched within the scope of this updater.
   *
   * @return the accounts that have been touched within the scope of this updater
   */
  Collection<? extends Account> getTouchedAccounts();

  /**
   * Returns the account addresses that have been deleted within the scope of this updater.
   *
   * @return the account addresses that have been deleted within the scope of this updater
   */
  Collection<Address> getDeletedAccountAddresses();

  /** Removes the changes that were made to this updater. */
  void revert();

  /**
   * Commits the changes made to this updater to the underlying {@link WorldView} this is an updater
   * of.
   */
  void commit();

  /**
   * The parent updater (if it exists).
   *
   * @return The parent WorldUpdater if this wraps another one, empty otherwise
   */
  Optional<WorldUpdater> parentUpdater();
}

this line doesn't match what the rest says